From bac150f056c86feb4eb1e49ba57e47501093a6b0 Mon Sep 17 00:00:00 2001 From: Edward Welbourne Date: Mon, 16 Aug 2021 17:43:00 +0200 Subject: Add a repeated-failure selftest for testlib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This mimics the effects of test-code delegating checks to other functions, which can lead to repeated failures. The failing test should only contribute one to the total of failed tsts. Drive-by: correct an existing test's reporting of its own name. Task-number: QTBUG-95661 Change-Id: I370fd2aee378d9fdd35826f0d11677483c60423d Reviewed-by: Mårten Nordheim --- .../testlib/selftests/expected_subtest.junitxml | 17 ++-- .../testlib/selftests/expected_subtest.lightxml | 50 +++++++++++- tests/auto/testlib/selftests/expected_subtest.tap | 95 ++++++++++++++++++++-- .../testlib/selftests/expected_subtest.teamcity | 32 +++++++- tests/auto/testlib/selftests/expected_subtest.txt | 34 ++++++-- tests/auto/testlib/selftests/expected_subtest.xml | 50 +++++++++++- .../auto/testlib/selftests/subtest/tst_subtest.cpp | 18 +++- 7 files changed, 263 insertions(+), 33 deletions(-) (limited to 'tests/auto/testlib') diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml index 72b2c5c6b5..353f739985 100644 --- a/tests/auto/testlib/selftests/expected_subtest.junitxml +++ b/tests/auto/testlib/selftests/expected_subtest.junitxml @@ -1,5 +1,5 @@ - + @@ -48,8 +48,8 @@ - - + + @@ -60,7 +60,7 @@ - + @@ -71,10 +71,17 @@ - + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_subtest.lightxml b/tests/auto/testlib/selftests/expected_subtest.lightxml index f35f6a6dc4..2c98c13cda 100644 --- a/tests/auto/testlib/selftests/expected_subtest.lightxml +++ b/tests/auto/testlib/selftests/expected_subtest.lightxml @@ -102,11 +102,11 @@ - + - + @@ -121,7 +121,7 @@ - + @@ -139,7 +139,7 @@ - + @@ -153,6 +153,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_subtest.tap b/tests/auto/testlib/selftests/expected_subtest.tap index 395eb2776c..ebbc1d1067 100644 --- a/tests/auto/testlib/selftests/expected_subtest.tap +++ b/tests/auto/testlib/selftests/expected_subtest.tap @@ -26,12 +26,12 @@ ok 5 - test2(data2) # test3_data test3 (null) # test3_data end # init test3 data0 -# test2 test3 data0 -# test2 end +# test3 test3 data0 +# test3 end # cleanup test3 data0 ok 6 - test3(data0) # init test3 data1 -# test2 test3 data1 +# test3 test3 data1 not ok 7 - test3(data1) --- type: QCOMPARE @@ -46,7 +46,7 @@ not ok 7 - test3(data1) ... # cleanup test3 data1 # init test3 data2 -# test2 test3 data2 +# test3 test3 data2 not ok 8 - test3(data2) --- type: QCOMPARE @@ -60,9 +60,88 @@ not ok 8 - test3(data2) line: 0 ... # cleanup test3 data2 +# init multiFail (null) +not ok 9 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 10 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 11 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 12 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 13 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 14 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 15 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 16 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 17 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 18 - multiFail() + --- + # This failure message should be repeated ten times + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +not ok 19 - multiFail() + --- + # But this test should only contribute one to the failure count + at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp + line: 0 + ... +# cleanup multiFail (null) # cleanupTestCase cleanupTestCase (null) -ok 9 - cleanupTestCase() -1..9 -# tests 9 +ok 20 - cleanupTestCase() +1..20 +# tests 20 # pass 7 -# fail 2 +# fail 13 diff --git a/tests/auto/testlib/selftests/expected_subtest.teamcity b/tests/auto/testlib/selftests/expected_subtest.teamcity index 3fcbcb7ea4..580e5695dd 100644 --- a/tests/auto/testlib/selftests/expected_subtest.teamcity +++ b/tests/auto/testlib/selftests/expected_subtest.teamcity @@ -15,17 +15,41 @@ ##teamcity[testStdOut name='test2(data2)' out='QDEBUG: init test2 data2|nQDEBUG: test2 test2 data2|nQDEBUG: test2 end|nQDEBUG: cleanup test2 data2' flowId='tst_Subtest'] ##teamcity[testFinished name='test2(data2)' flowId='tst_Subtest'] ##teamcity[testStarted name='test3(data0)' flowId='tst_Subtest'] -##teamcity[testStdOut name='test3(data0)' out='QDEBUG: test3_data test3 (null)|nQDEBUG: test3_data end|nQDEBUG: init test3 data0|nQDEBUG: test2 test3 data0|nQDEBUG: test2 end|nQDEBUG: cleanup test3 data0' flowId='tst_Subtest'] +##teamcity[testStdOut name='test3(data0)' out='QDEBUG: test3_data test3 (null)|nQDEBUG: test3_data end|nQDEBUG: init test3 data0|nQDEBUG: test3 test3 data0|nQDEBUG: test3 end|nQDEBUG: cleanup test3 data0' flowId='tst_Subtest'] ##teamcity[testFinished name='test3(data0)' flowId='tst_Subtest'] ##teamcity[testStarted name='test3(data1)' flowId='tst_Subtest'] ##teamcity[testFailed name='test3(data1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello1"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest'] -##teamcity[testStdOut name='test3(data1)' out='QDEBUG: init test3 data1|nQDEBUG: test2 test3 data1' flowId='tst_Subtest'] +##teamcity[testStdOut name='test3(data1)' out='QDEBUG: init test3 data1|nQDEBUG: test3 test3 data1' flowId='tst_Subtest'] ##teamcity[testFinished name='test3(data1)' flowId='tst_Subtest'] ##teamcity[testStarted name='test3(data2)' flowId='tst_Subtest'] ##teamcity[testFailed name='test3(data2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello2"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest'] -##teamcity[testStdOut name='test3(data2)' out='QDEBUG: cleanup test3 data1|nQDEBUG: init test3 data2|nQDEBUG: test2 test3 data2' flowId='tst_Subtest'] +##teamcity[testStdOut name='test3(data2)' out='QDEBUG: cleanup test3 data1|nQDEBUG: init test3 data2|nQDEBUG: test3 test3 data2' flowId='tst_Subtest'] ##teamcity[testFinished name='test3(data2)' flowId='tst_Subtest'] +##teamcity[testStarted name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testStdOut name='multiFail()' out='QDEBUG: cleanup test3 data2|nQDEBUG: init multiFail (null)' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='But this test should only contribute one to the failure count' flowId='tst_Subtest'] +##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest'] -##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup test3 data2|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest'] ##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest'] diff --git a/tests/auto/testlib/selftests/expected_subtest.txt b/tests/auto/testlib/selftests/expected_subtest.txt index 8b0dbf63ee..d35d7b4d24 100644 --- a/tests/auto/testlib/selftests/expected_subtest.txt +++ b/tests/auto/testlib/selftests/expected_subtest.txt @@ -26,25 +26,49 @@ PASS : tst_Subtest::test2(data2) QDEBUG : tst_Subtest::test3() test3_data test3 (null) QDEBUG : tst_Subtest::test3() test3_data end QDEBUG : tst_Subtest::test3(data0) init test3 data0 -QDEBUG : tst_Subtest::test3(data0) test2 test3 data0 -QDEBUG : tst_Subtest::test3(data0) test2 end +QDEBUG : tst_Subtest::test3(data0) test3 test3 data0 +QDEBUG : tst_Subtest::test3(data0) test3 end QDEBUG : tst_Subtest::test3(data0) cleanup test3 data0 PASS : tst_Subtest::test3(data0) QDEBUG : tst_Subtest::test3(data1) init test3 data1 -QDEBUG : tst_Subtest::test3(data1) test2 test3 data1 +QDEBUG : tst_Subtest::test3(data1) test3 test3 data1 FAIL! : tst_Subtest::test3(data1) Compared values are not the same Actual (str) : "hello1" Expected (QString("hello0")): "hello0" Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] QDEBUG : tst_Subtest::test3(data1) cleanup test3 data1 QDEBUG : tst_Subtest::test3(data2) init test3 data2 -QDEBUG : tst_Subtest::test3(data2) test2 test3 data2 +QDEBUG : tst_Subtest::test3(data2) test3 test3 data2 FAIL! : tst_Subtest::test3(data2) Compared values are not the same Actual (str) : "hello2" Expected (QString("hello0")): "hello0" Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] QDEBUG : tst_Subtest::test3(data2) cleanup test3 data2 +QDEBUG : tst_Subtest::multiFail() init multiFail (null) +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +FAIL! : tst_Subtest::multiFail() But this test should only contribute one to the failure count + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +QDEBUG : tst_Subtest::multiFail() cleanup multiFail (null) QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null) PASS : tst_Subtest::cleanupTestCase() -Totals: 7 passed, 2 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 7 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Subtest ********* diff --git a/tests/auto/testlib/selftests/expected_subtest.xml b/tests/auto/testlib/selftests/expected_subtest.xml index 793f5d4d2d..1f3867de3a 100644 --- a/tests/auto/testlib/selftests/expected_subtest.xml +++ b/tests/auto/testlib/selftests/expected_subtest.xml @@ -104,11 +104,11 @@ - + - + @@ -123,7 +123,7 @@ - + @@ -141,7 +141,7 @@ - + @@ -155,6 +155,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp index 0837e70233..416cf80aa8 100644 --- a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp +++ b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the test suite of the Qt Toolkit. @@ -47,6 +47,8 @@ private slots: void test2(); void test3_data(); void test3(); + + void multiFail(); }; @@ -131,7 +133,7 @@ void tst_Subtest::test3_data() void tst_Subtest::test3() { - qDebug() << "test2" + qDebug() << "test3" << (QTest::currentTestFunction() ? QTest::currentTestFunction() : "(null)") << (QTest::currentDataTag() ? QTest::currentDataTag() : "(null)"); @@ -140,7 +142,17 @@ void tst_Subtest::test3() // second and third time we call this it should FAIL QCOMPARE(str, QString("hello0")); - qDebug() << "test2 end"; + qDebug() << "test3 end"; +} + +void tst_Subtest::multiFail() +{ + // Simulates tests which call a shared function that does common checks, or + // that do checks in code run asynchronously from a messae loop. + for (int i = 0; i < 10; ++i) + []() { QFAIL("This failure message should be repeated ten times"); }(); + // FIXME QTBUG-95661: it gets counted as eleven failures, of course. + QFAIL("But this test should only contribute one to the failure count"); } QTEST_MAIN(tst_Subtest) -- cgit v1.2.3