diff options
| author | Friedemann Kleint <Friedemann.Kleint@qt.io> | 2021-01-13 11:21:01 +0100 |
|---|---|---|
| committer | Friedemann Kleint <Friedemann.Kleint@qt.io> | 2021-01-15 18:05:40 +0100 |
| commit | 539553a57216c4ece292ff54fb623807c334cdb6 (patch) | |
| tree | 55c4c80b5fe764717dbe32169dd1906a5cf6bcb5 /tests/auto/testlib | |
| parent | 7f5d41e286e043c5f3061e2eb5a0128102a38c80 (diff) | |
QTestlib/JUnit XML: Log most messages to standard output instead of error output
Add an enumeration for system-out and alog element
for it. Redirect the messages types that are not warnings/errors
to this element. For compatibility, write it out only
if it is not empty. Rename enumerations and members accordingly.
[ChangeLog][QtTestLib] In JUnit XML, output that is
not a warning/error is now logged under <system-out>
instead of <system-err>.
Fixes: QTBUG-86540
Change-Id: I55598eafa7dafa486ac5a8221029c332ff47413b
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Diffstat (limited to 'tests/auto/testlib')
23 files changed, 79 insertions, 48 deletions
diff --git a/tests/auto/testlib/selftests/expected_badxml.junitxml b/tests/auto/testlib/selftests/expected_badxml.junitxml index 0781f1c92e..75a08ead6c 100644 --- a/tests/auto/testlib/selftests/expected_badxml.junitxml +++ b/tests/auto/testlib/selftests/expected_badxml.junitxml @@ -33,7 +33,7 @@ <!-- type="qdebug" message=""Ülrich Ümläut"" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[a message]]> <![CDATA[a message]]> <![CDATA[a message]]> @@ -47,5 +47,6 @@ <![CDATA[xml close > open < tags < text]]> <![CDATA[all > " mixed ]]]><![CDATA[]> up > " in < the ]]]><![CDATA[]> hopes < of triggering "< ]]]><![CDATA[]> bugs]]> <![CDATA["Ülrich Ümläut"]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml b/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml index 09e3de7ef0..1766bbd5b2 100644 --- a/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml @@ -15,7 +15,8 @@ <failure result="fail" message="This is a failing benchmark"/> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[This is a skipping benchmark]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index a9779140fb..9d6131403e 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -22,14 +22,16 @@ <!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" --> <!-- type="qfatal" message="This is a fatal error message that should still appear in silent test output" --> </testcase> - <system-err> + <system-out> <![CDATA[This test should SKIP]]> -<![CDATA[This is a warning that should not appear in silent test output]]> -<![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a debug message that should not appear in silent test output]]> <![CDATA[This is a critical message that should not appear in silent test output]]> <![CDATA[This is an info message that should not appear in silent test output]]> <![CDATA[This is an internal testlib info message that should not appear in silent test output]]> + </system-out> + <system-err> +<![CDATA[This is a warning that should not appear in silent test output]]> +<![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a fatal error message that should still appear in silent test output]]> </system-err> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_commandlinedata.junitxml b/tests/auto/testlib/selftests/expected_commandlinedata.junitxml index e5cd575d34..170a461075 100644 --- a/tests/auto/testlib/selftests/expected_commandlinedata.junitxml +++ b/tests/auto/testlib/selftests/expected_commandlinedata.junitxml @@ -17,12 +17,13 @@ <!-- type="info" message="QVERIFY(test)" tag="fiveTablePasses_data1" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[QVERIFY(test)]]> <![CDATA[QVERIFY(test)]]> <![CDATA[QVERIFY(test)]]> <![CDATA[QVERIFY(test)]]> <![CDATA[QVERIFY(test)]]> <![CDATA[QVERIFY(test)]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_counting.junitxml b/tests/auto/testlib/selftests/expected_counting.junitxml index 49632731d0..a9278ad0a4 100644 --- a/tests/auto/testlib/selftests/expected_counting.junitxml +++ b/tests/auto/testlib/selftests/expected_counting.junitxml @@ -50,7 +50,7 @@ <!-- type="skip" message="Skip in cleanup()" tag="skip" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[Skipping]]> <![CDATA[Skipping]]> <![CDATA[Skipping]]> @@ -61,5 +61,6 @@ <![CDATA[Skip in init()]]> <![CDATA[This test function should execute and then QSKIP in cleanup()]]> <![CDATA[Skip in cleanup()]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index e38933a3df..6b70eab354 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -49,7 +49,7 @@ <failure result="xpass" message="QCOMPARE(1, 1) returned TRUE unexpectedly." tag="XPass"/> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[begin]]> <![CDATA[This should xfail]]> <![CDATA[after]]> @@ -63,5 +63,6 @@ <![CDATA[This test should xfail]]> <![CDATA[This test should xfail]]> <![CDATA[This test should xfail]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_faildatatype.junitxml b/tests/auto/testlib/selftests/expected_faildatatype.junitxml index e654fa4441..9ca9a69030 100644 --- a/tests/auto/testlib/selftests/expected_faildatatype.junitxml +++ b/tests/auto/testlib/selftests/expected_faildatatype.junitxml @@ -11,8 +11,10 @@ <!-- type="qfatal" message="ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0" --> <failure result="fail" message="Received a fatal error."/> </testcase> - <system-err> + <system-out> <![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]> + </system-out> + <system-err> <![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]> </system-err> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_float.junitxml b/tests/auto/testlib/selftests/expected_float.junitxml index 831f8fefd8..c298974967 100644 --- a/tests/auto/testlib/selftests/expected_float.junitxml +++ b/tests/auto/testlib/selftests/expected_float.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_float" timestamp="@TEST_START_TIME@" tests="6" failures="96" errors="0" time="@TEST_DURATION@"> +<testsuite name="tst_float" timestamp="@TEST_START_TIME@" tests="6" failures="97" errors="0" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> @@ -201,12 +201,15 @@ Actual (operandLeft) : 1 Expected (operandRight): 3" tag="should FAIL 1"/> <failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare) - Actual (operandLeft) : 0.000999 + Actual (operandLeft) : 0.001 Expected (operandRight): 0.003" tag="should FAIL 2"/> <failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare) Actual (operandLeft) : 98 Expected (operandRight): 99" tag="should FAIL 3"/> <failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 0.001 + Expected (operandRight): -0.001" tag="should PASS 3"/> + <failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare) Actual (operandLeft) : 0.00101 Expected (operandRight): 0.00099" tag="should FAIL 4"/> <failure result="fail" message="Compared qfloat16s are not the same (fuzzy compare) diff --git a/tests/auto/testlib/selftests/expected_globaldata.junitxml b/tests/auto/testlib/selftests/expected_globaldata.junitxml index 6a24fd099b..d371a5affd 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.junitxml +++ b/tests/auto/testlib/selftests/expected_globaldata.junitxml @@ -60,7 +60,7 @@ <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"> <!-- type="qdebug" message="cleanupTestCase cleanupTestCase (null)" --> </testcase> - <system-err> + <system-out> <![CDATA[initTestCase initTestCase (null)]]> <![CDATA[init testGlobal local=false]]> <![CDATA[global: false]]> @@ -104,5 +104,6 @@ <![CDATA[global: true local: true]]> <![CDATA[cleanup skipSingle local=true]]> <![CDATA[cleanupTestCase cleanupTestCase (null)]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_maxwarnings.junitxml b/tests/auto/testlib/selftests/expected_maxwarnings.junitxml index def32cdb16..02d84648dc 100644 --- a/tests/auto/testlib/selftests/expected_maxwarnings.junitxml +++ b/tests/auto/testlib/selftests/expected_maxwarnings.junitxml @@ -2011,6 +2011,9 @@ <!-- type="system" message="Maximum amount of warnings exceeded. Use -maxwarnings to override." --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> + <system-out> +<![CDATA[Maximum amount of warnings exceeded. Use -maxwarnings to override.]]> + </system-out> <system-err> <![CDATA[0]]> <![CDATA[1]]> @@ -4013,6 +4016,5 @@ <![CDATA[1998]]> <![CDATA[1999]]> <![CDATA[2000]]> -<![CDATA[Maximum amount of warnings exceeded. Use -maxwarnings to override.]]> </system-err> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_signaldumper.junitxml b/tests/auto/testlib/selftests/expected_signaldumper.junitxml index a6f34414f8..b1aa68cc67 100644 --- a/tests/auto/testlib/selftests/expected_signaldumper.junitxml +++ b/tests/auto/testlib/selftests/expected_signaldumper.junitxml @@ -153,7 +153,7 @@ <!-- type="info" message="Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()]]> <![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithParameters (int(242), char(m))]]> <![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()]]> @@ -278,5 +278,6 @@ <![CDATA[Signal: SignalSlotClass(_POINTER_) qVariantSignal (QVariant())]]> <![CDATA[Signal: SignalSlotClass(_POINTER_) qVariantSignal (QVariant())]]> <![CDATA[Signal: SignalSlotClass(_POINTER_) signalWithoutParameters ()]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_silent.junitxml b/tests/auto/testlib/selftests/expected_silent.junitxml index 10d8151d2e..c39fee5bf7 100644 --- a/tests/auto/testlib/selftests/expected_silent.junitxml +++ b/tests/auto/testlib/selftests/expected_silent.junitxml @@ -29,15 +29,17 @@ <!-- type="qfatal" message="This is a fatal error message that should still appear in silent test output" --> <failure result="fail" message="Received a fatal error."/> </testcase> - <system-err> + <system-out> <![CDATA[This test should skip]]> <![CDATA[This test should XFAIL]]> -<![CDATA[This is a warning that should not appear in silent test output]]> -<![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a debug message that should not appear in silent test output]]> <![CDATA[This is a critical message that should not appear in silent test output]]> <![CDATA[This is an info message that should not appear in silent test output]]> <![CDATA[This is an internal testlib info message that should not appear in silent test output]]> + </system-out> + <system-err> +<![CDATA[This is a warning that should not appear in silent test output]]> +<![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a fatal error message that should still appear in silent test output]]> </system-err> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_singleskip.junitxml b/tests/auto/testlib/selftests/expected_singleskip.junitxml index a2568ca553..2949553671 100644 --- a/tests/auto/testlib/selftests/expected_singleskip.junitxml +++ b/tests/auto/testlib/selftests/expected_singleskip.junitxml @@ -10,7 +10,8 @@ <!-- type="skip" message="skipping test" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[skipping test]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_skip.junitxml b/tests/auto/testlib/selftests/expected_skip.junitxml index b82b176ecc..f4213a723f 100644 --- a/tests/auto/testlib/selftests/expected_skip.junitxml +++ b/tests/auto/testlib/selftests/expected_skip.junitxml @@ -17,10 +17,11 @@ <!-- type="qdebug" message="this line should only be reached once (true)" tag="local 2" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[skipping all]]> <![CDATA[skipping all]]> <![CDATA[skipping one]]> <![CDATA[this line should only be reached once (true)]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml index cb45485b4b..a7d7fd0bb4 100644 --- a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml +++ b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml @@ -10,7 +10,8 @@ <testcase name="cleanupTestCase" time="@TEST_DURATION@"> <!-- type="skip" message="Skip inside cleanupTestCase." --> </testcase> - <system-err> + <system-out> <![CDATA[Skip inside cleanupTestCase.]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_skipinit.junitxml b/tests/auto/testlib/selftests/expected_skipinit.junitxml index 9bdb22230a..2213d11cfe 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.junitxml +++ b/tests/auto/testlib/selftests/expected_skipinit.junitxml @@ -9,7 +9,8 @@ <!-- type="skip" message="Skip inside initTestCase. This should skip all tests in the class." --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.junitxml b/tests/auto/testlib/selftests/expected_skipinitdata.junitxml index 56ffbcba86..4ae6af2399 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.junitxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.junitxml @@ -8,7 +8,8 @@ <testcase name="initTestCase" time="@TEST_DURATION@"> <!-- type="skip" message="Skip inside initTestCase_data. This should skip all tests in the class." --> </testcase> - <system-err> + <system-out> <![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_strcmp.junitxml b/tests/auto/testlib/selftests/expected_strcmp.junitxml index 9d51829f4f..6d87553356 100644 --- a/tests/auto/testlib/selftests/expected_strcmp.junitxml +++ b/tests/auto/testlib/selftests/expected_strcmp.junitxml @@ -36,9 +36,10 @@ Expected (QByteArray("7")): "7""/> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[Next test should fail]]> <![CDATA[Next test should fail]]> <![CDATA[Next test should fail]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml index 27996a4992..36a919d5b2 100644 --- a/tests/auto/testlib/selftests/expected_subtest.junitxml +++ b/tests/auto/testlib/selftests/expected_subtest.junitxml @@ -52,7 +52,7 @@ <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"> <!-- type="qdebug" message="cleanupTestCase cleanupTestCase (null)" --> </testcase> - <system-err> + <system-out> <![CDATA[initTestCase initTestCase (null)]]> <![CDATA[init test1 (null)]]> <![CDATA[test1 test1 (null)]]> @@ -84,5 +84,6 @@ <![CDATA[test2 test3 data2]]> <![CDATA[cleanup test3 data2]]> <![CDATA[cleanupTestCase cleanupTestCase (null)]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_verbose1.junitxml b/tests/auto/testlib/selftests/expected_verbose1.junitxml index 49632731d0..a9278ad0a4 100644 --- a/tests/auto/testlib/selftests/expected_verbose1.junitxml +++ b/tests/auto/testlib/selftests/expected_verbose1.junitxml @@ -50,7 +50,7 @@ <!-- type="skip" message="Skip in cleanup()" tag="skip" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[Skipping]]> <![CDATA[Skipping]]> <![CDATA[Skipping]]> @@ -61,5 +61,6 @@ <![CDATA[Skip in init()]]> <![CDATA[This test function should execute and then QSKIP in cleanup()]]> <![CDATA[Skip in cleanup()]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_verbose2.junitxml b/tests/auto/testlib/selftests/expected_verbose2.junitxml index 88dd82f620..408c71425c 100644 --- a/tests/auto/testlib/selftests/expected_verbose2.junitxml +++ b/tests/auto/testlib/selftests/expected_verbose2.junitxml @@ -69,7 +69,7 @@ <!-- type="skip" message="Skip in cleanup()" tag="skip" --> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> + <system-out> <![CDATA[QVERIFY(true)]]> <![CDATA[QCOMPARE(2 + 1, 3)]]> <![CDATA[QVERIFY(true)]]> @@ -98,5 +98,6 @@ <![CDATA[Skip in init()]]> <![CDATA[This test function should execute and then QSKIP in cleanup()]]> <![CDATA[Skip in cleanup()]]> - </system-err> + </system-out> + <system-err/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml index 0a0a8aec26..8d49705972 100644 --- a/tests/auto/testlib/selftests/expected_warnings.junitxml +++ b/tests/auto/testlib/selftests/expected_warnings.junitxml @@ -36,9 +36,7 @@ <failure result="fail" message="Not all expected messages were received" tag="second row"/> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> -<![CDATA[Warning]]> -<![CDATA[Warning]]> + <system-out> <![CDATA[Debug]]> <![CDATA[Debug]]> <![CDATA[Info]]> @@ -46,7 +44,6 @@ <![CDATA[Baba]]> <![CDATA[Baba]]> <![CDATA[Bubublabla]]> -<![CDATA[Babablabla]]> <![CDATA[Did not receive message: "Warning0"]]> <![CDATA[Did not receive message: "Warning1"]]> <![CDATA[Did not receive any message matching: "Warning\s\d"]]> @@ -54,5 +51,10 @@ <![CDATA[Did not receive message: "Warning1"]]> <![CDATA[Did not receive message: "Warning0"]]> <![CDATA[Did not receive message: "Warning1"]]> + </system-out> + <system-err> +<![CDATA[Warning]]> +<![CDATA[Warning]]> +<![CDATA[Babablabla]]> </system-err> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_xunit.junitxml b/tests/auto/testlib/selftests/expected_xunit.junitxml index 817f85e7f0..e1b21760f2 100644 --- a/tests/auto/testlib/selftests/expected_xunit.junitxml +++ b/tests/auto/testlib/selftests/expected_xunit.junitxml @@ -31,11 +31,13 @@ <failure result="xpass" message="'true' returned TRUE unexpectedly. ()"/> </testcase> <testcase name="cleanupTestCase" result="pass" time="@TEST_DURATION@"/> - <system-err> -<![CDATA[just a QWARN() !]]> + <system-out> <![CDATA[a qDebug() call with comment-ending stuff -->]]> <![CDATA[skipping this function!]]> <![CDATA[this failure is expected]]> <![CDATA[this failure is also expected]]> + </system-out> + <system-err> +<![CDATA[just a QWARN() !]]> </system-err> </testsuite> |