diff options
Diffstat (limited to 'tests/auto/testlib')
21 files changed, 369 insertions, 47 deletions
diff --git a/tests/auto/testlib/outformat/outformat.pro b/tests/auto/testlib/outformat/outformat.pro new file mode 100644 index 0000000000..ea02f3167f --- /dev/null +++ b/tests/auto/testlib/outformat/outformat.pro @@ -0,0 +1,7 @@ +CONFIG += testcase +QT = core testlib + +SOURCES += tst_outformat.cpp +TARGET = outformat + +include($$QT_SOURCE_TREE/src/testlib/selfcover.pri) diff --git a/tests/auto/testlib/outformat/tst_outformat.cpp b/tests/auto/testlib/outformat/tst_outformat.cpp new file mode 100644 index 0000000000..5d131159a9 --- /dev/null +++ b/tests/auto/testlib/outformat/tst_outformat.cpp @@ -0,0 +1,71 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include <QtTest/QtTest> + +class tst_OutFormat : public QObject +{ + Q_OBJECT +private slots: + void toHex_data() const; + void toHex() const; + // other formats of interest ? +}; + +void tst_OutFormat::toHex_data() const +{ + QTest::addColumn<QByteArray>("raw"); + QTest::addColumn<QByteArray>("hex"); + + QTest::newRow("empty") << QByteArray("") << QByteArray(""); + QTest::newRow("long") + << QByteArray("Truncates in ellipsis when more than fifty characters long") + << QByteArray("54 72 75 6E 63 61 74 65 73 20 69 6E 20 65 6C 6C " + "69 70 73 69 73 20 77 68 65 6E 20 6D 6F 72 65 20 " + "74 68 61 6E 20 66 69 66 74 79 20 63 68 61 72 61 " + "63 74 ..."); + QTest::newRow("spaces") + << QByteArray(" \t\n\v\f\r") << QByteArray("20 09 0A 0B 0C 0D"); + QTest::newRow("ASCII-escapes") + << QByteArray("\a\b\\\"'\177") << QByteArray("07 08 5C 22 27 7F"); + // These are the ISO Latin-15 , pound, Euro, ..., y-umlaut + QTest::newRow("8-bit-sampler") + << QByteArray("\240\243\244\261\327\360\377") << QByteArray("A0 A3 A4 B1 D7 F0 FF"); +} + +void tst_OutFormat::toHex() const +{ + QFETCH(QByteArray, raw); + QFETCH(QByteArray, hex); + QScopedArrayPointer<char> repr(QTest::toHexRepresentation(raw.constData(), raw.size())); + QCOMPARE(repr.data(), hex); +} + +QTEST_APPLESS_MAIN(tst_OutFormat) + +#include "tst_outformat.moc" diff --git a/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp b/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp index 0593ae74bf..60aa350145 100644 --- a/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp +++ b/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp @@ -79,9 +79,9 @@ void tst_QAbstractItemModelTester::treeWidgetModel() root->removeChild(remove); QTreeWidgetItem *parent = new QTreeWidgetItem(&widget, QStringList("parent")); new QTreeWidgetItem(parent, QStringList("child")); - widget.setItemHidden(parent, true); + parent->setHidden(true); - widget.sortByColumn(0); + widget.sortByColumn(0, Qt::AscendingOrder); } void tst_QAbstractItemModelTester::standardItemModel() diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index 36b7699cbd..a923c11416 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -1,12 +1,20 @@ -[pass] +obscure # no such platform; is ignored * + +[pass] +!* + [skip] * + [fail] * -[xpass] -* + [xfail] * + +[xpass] +* + [messages] * diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index 8578752e22..90520385ec 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -45,7 +45,8 @@ private slots: void messages(); }; -// All the tests below have been blacklisted in blacklisted/BLACKLIST +// All the tests below except pass() have been blacklisted in blacklisted/BLACKLIST +// Contrast with ../silent/, for the same tests without blacklisting but with -silent void tst_Blacklisted::pass() { @@ -64,7 +65,7 @@ void tst_Blacklisted::fail() void tst_Blacklisted::xfail() { - QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort); + QEXPECT_FAIL("", "This test should XFAIL then BPASS", Abort); QVERIFY(false); } diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index 4193628e7c..98d7a38ca9 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -25,7 +25,7 @@ </TestFunction> <TestFunction name="xfail"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> - <Description><![CDATA[This test should XFAIL then BFAIL]]></Description> + <Description><![CDATA[This test should XFAIL then BPASS]]></Description> </Incident> <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap new file mode 100644 index 0000000000..f26155ded0 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -0,0 +1,43 @@ +TAP version 13 +# tst_Blacklisted +ok 1 - initTestCase() +ok 2 - pass() # TODO +ok 3 - skip() # SKIP This test should SKIP +not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL) + --- + type: QVERIFY + message: This test should BFAIL + wanted: true (false) + found: false (false) + expected: true (false) + actual: false (false) + at: tst_Blacklisted::fail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:63) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 63 + ... +not ok 5 - xfail() # TODO This test should XFAIL then BPASS + --- + at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:69) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 69 + ... +ok 5 - xfail() # TODO +ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS) +# This is a warning that should not appear in silent test output +# This is an internal testlib warning that should not appear in silent test output +# This is a debug message that should not appear in silent test output +# This is a critical message that should not appear in silent test output +# This is an info message that should not appear in silent test output +# This is an internal testlib info message that should not appear in silent test output +# This is a fatal error message that should still appear in silent test output +not ok 7 - messages() # TODO Received a fatal error. + --- + # Received a fatal error. + at: tst_Blacklisted::messages() (Unknown file:0) + file: Unknown file + line: 0 + ... +1..7 +# tests 7 +# pass 1 +# fail 1 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index 8180a7ce76..df58208fb3 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -7,7 +7,7 @@ ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='fail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted'] -##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BPASS' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted'] ##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index 6fa2403b59..3cfe40eb12 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -6,7 +6,7 @@ SKIP : tst_Blacklisted::skip() This test should SKIP Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BFAIL +XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BPASS Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BPASS : tst_Blacklisted::xfail() XPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS) diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index 443bc6b199..f0387bae00 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -27,7 +27,7 @@ </TestFunction> <TestFunction name="xfail"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> - <Description><![CDATA[This test should XFAIL then BFAIL]]></Description> + <Description><![CDATA[This test should XFAIL then BPASS]]></Description> </Incident> <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml index 2752bc18b4..e2d0cd009b 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml @@ -12,7 +12,7 @@ </testcase> <testcase result="bfail" name="fail"/> <testcase result="xfail" name="xfail"> - <!-- message="This test should XFAIL then BFAIL" type="info" --> + <!-- message="This test should XFAIL then BPASS" type="info" --> </testcase> <testcase result="xpass" name="xpass"> <failure message="'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/> @@ -28,7 +28,7 @@ </testcase> <system-err> <![CDATA[This test should SKIP]]> -<![CDATA[This test should XFAIL then BFAIL]]> +<![CDATA[This test should XFAIL then BPASS]]> <![CDATA[This is a warning that should not appear in silent test output]]> <![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a debug message that should not appear in silent test output]]> diff --git a/tests/auto/testlib/selftests/expected_float.lightxml b/tests/auto/testlib/selftests/expected_float.lightxml index 0dbc5dd8c8..37ccbfc2c5 100644 --- a/tests/auto/testlib/selftests/expected_float.lightxml +++ b/tests/auto/testlib/selftests/expected_float.lightxml @@ -34,6 +34,33 @@ </Incident> <Duration msecs="0"/> </TestFunction> +<TestFunction name="float16Comparisons"> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[should SUCCEED 1]]></DataTag> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> + <DataTag><![CDATA[should FAIL 1]]></DataTag> + <Description><![CDATA[Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 1 + Expected (operandRight): 3]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> + <DataTag><![CDATA[should FAIL 2]]></DataTag> + <Description><![CDATA[Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 0.0001 + Expected (operandRight): 0.0003]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> + <DataTag><![CDATA[should FAIL 3]]></DataTag> + <Description><![CDATA[Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 98 + Expected (operandRight): 99]]></Description> +</Incident> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[should SUCCEED 2]]></DataTag> +</Incident> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="compareFloatTests"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> <DataTag><![CDATA[1e0]]></DataTag> diff --git a/tests/auto/testlib/selftests/expected_float.tap b/tests/auto/testlib/selftests/expected_float.tap index fae2dc9796..158eff1470 100644 --- a/tests/auto/testlib/selftests/expected_float.tap +++ b/tests/auto/testlib/selftests/expected_float.tap @@ -10,9 +10,9 @@ not ok 3 - floatComparisons(should FAIL 1) found: 1 (operandLeft) expected: 3 (operandRight) actual: 1 (operandLeft) - at: tst_float::floatComparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:48) + at: tst_float::floatComparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:51) file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp - line: 48 + line: 51 ... not ok 4 - floatComparisons(should FAIL 2) --- @@ -22,9 +22,9 @@ not ok 4 - floatComparisons(should FAIL 2) found: 1e-07 (operandLeft) expected: 3e-07 (operandRight) actual: 1e-07 (operandLeft) - at: tst_float::floatComparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:48) + at: tst_float::floatComparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:51) file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp - line: 48 + line: 51 ... not ok 5 - floatComparisons(should FAIL 3) --- @@ -34,12 +34,50 @@ not ok 5 - floatComparisons(should FAIL 3) found: 99998 (operandLeft) expected: 99999 (operandRight) actual: 99998 (operandLeft) - at: tst_float::floatComparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:48) + at: tst_float::floatComparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:51) file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp - line: 48 + line: 51 ... ok 6 - floatComparisons(should SUCCEED 2) -not ok 7 - compareFloatTests(1e0) +ok 7 - float16Comparisons(should SUCCEED 1) +not ok 8 - float16Comparisons(should FAIL 1) + --- + type: QCOMPARE + message: Compared qfloat16s are not the same (fuzzy compare) + wanted: 3 (operandRight) + found: 1 (operandLeft) + expected: 3 (operandRight) + actual: 1 (operandLeft) + at: tst_float::float16Comparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:90) + file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp + line: 90 + ... +not ok 9 - float16Comparisons(should FAIL 2) + --- + type: QCOMPARE + message: Compared qfloat16s are not the same (fuzzy compare) + wanted: 0.0003 (operandRight) + found: 0.0001 (operandLeft) + expected: 0.0003 (operandRight) + actual: 0.0001 (operandLeft) + at: tst_float::float16Comparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:90) + file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp + line: 90 + ... +not ok 10 - float16Comparisons(should FAIL 3) + --- + type: QCOMPARE + message: Compared qfloat16s are not the same (fuzzy compare) + wanted: 99 (operandRight) + found: 98 (operandLeft) + expected: 99 (operandRight) + actual: 98 (operandLeft) + at: tst_float::float16Comparisons() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:90) + file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp + line: 90 + ... +ok 11 - float16Comparisons(should SUCCEED 2) +not ok 12 - compareFloatTests(1e0) --- type: QCOMPARE message: Compared floats are not the same (fuzzy compare) @@ -47,11 +85,11 @@ not ok 7 - compareFloatTests(1e0) found: 1 (t1) expected: 3 (t3) actual: 1 (t1) - at: tst_float::compareFloatTests() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:96) + at: tst_float::compareFloatTests() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:135) file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp - line: 96 + line: 135 ... -not ok 8 - compareFloatTests(1e-7) +not ok 13 - compareFloatTests(1e-7) --- type: QCOMPARE message: Compared floats are not the same (fuzzy compare) @@ -59,11 +97,11 @@ not ok 8 - compareFloatTests(1e-7) found: 1e-07 (t1) expected: 3e-07 (t3) actual: 1e-07 (t1) - at: tst_float::compareFloatTests() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:96) + at: tst_float::compareFloatTests() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:135) file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp - line: 96 + line: 135 ... -not ok 9 - compareFloatTests(1e+7) +not ok 14 - compareFloatTests(1e+7) --- type: QCOMPARE message: Compared floats are not the same (fuzzy compare) @@ -71,12 +109,12 @@ not ok 9 - compareFloatTests(1e+7) found: 1e+07 (t1) expected: 3e+07 (t3) actual: 1e+07 (t1) - at: tst_float::compareFloatTests() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:96) + at: tst_float::compareFloatTests() (qtbase/tests/auto/testlib/selftests/float/tst_float.cpp:135) file: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp - line: 96 + line: 135 ... -ok 10 - cleanupTestCase() -1..10 -# tests 10 -# pass 4 -# fail 6 +ok 15 - cleanupTestCase() +1..15 +# tests 15 +# pass 6 +# fail 9 diff --git a/tests/auto/testlib/selftests/expected_float.teamcity b/tests/auto/testlib/selftests/expected_float.teamcity index d5b81593d8..140dc7edf9 100644 --- a/tests/auto/testlib/selftests/expected_float.teamcity +++ b/tests/auto/testlib/selftests/expected_float.teamcity @@ -14,6 +14,19 @@ ##teamcity[testFinished name='floatComparisons(should FAIL 3)' flowId='tst_float'] ##teamcity[testStarted name='floatComparisons(should SUCCEED 2)' flowId='tst_float'] ##teamcity[testFinished name='floatComparisons(should SUCCEED 2)' flowId='tst_float'] +##teamcity[testStarted name='float16Comparisons(should SUCCEED 1)' flowId='tst_float'] +##teamcity[testFinished name='float16Comparisons(should SUCCEED 1)' flowId='tst_float'] +##teamcity[testStarted name='float16Comparisons(should FAIL 1)' flowId='tst_float'] +##teamcity[testFailed name='float16Comparisons(should FAIL 1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)|]' details='Compared qfloat16s are not the same (fuzzy compare)|n Actual (operandLeft) : 1|n Expected (operandRight): 3' flowId='tst_float'] +##teamcity[testFinished name='float16Comparisons(should FAIL 1)' flowId='tst_float'] +##teamcity[testStarted name='float16Comparisons(should FAIL 2)' flowId='tst_float'] +##teamcity[testFailed name='float16Comparisons(should FAIL 2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)|]' details='Compared qfloat16s are not the same (fuzzy compare)|n Actual (operandLeft) : 0.0001|n Expected (operandRight): 0.0003' flowId='tst_float'] +##teamcity[testFinished name='float16Comparisons(should FAIL 2)' flowId='tst_float'] +##teamcity[testStarted name='float16Comparisons(should FAIL 3)' flowId='tst_float'] +##teamcity[testFailed name='float16Comparisons(should FAIL 3)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)|]' details='Compared qfloat16s are not the same (fuzzy compare)|n Actual (operandLeft) : 98|n Expected (operandRight): 99' flowId='tst_float'] +##teamcity[testFinished name='float16Comparisons(should FAIL 3)' flowId='tst_float'] +##teamcity[testStarted name='float16Comparisons(should SUCCEED 2)' flowId='tst_float'] +##teamcity[testFinished name='float16Comparisons(should SUCCEED 2)' flowId='tst_float'] ##teamcity[testStarted name='compareFloatTests(1e0)' flowId='tst_float'] ##teamcity[testFailed name='compareFloatTests(1e0)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)|]' details='Compared floats are not the same (fuzzy compare)|n Actual (t1): 1|n Expected (t3): 3' flowId='tst_float'] ##teamcity[testFinished name='compareFloatTests(1e0)' flowId='tst_float'] diff --git a/tests/auto/testlib/selftests/expected_float.txt b/tests/auto/testlib/selftests/expected_float.txt index 8abea6a67e..4e3554758d 100644 --- a/tests/auto/testlib/selftests/expected_float.txt +++ b/tests/auto/testlib/selftests/expected_float.txt @@ -15,6 +15,20 @@ FAIL! : tst_float::floatComparisons(should FAIL 3) Compared floats are not the Expected (operandRight): 99999 Loc: [qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)] PASS : tst_float::floatComparisons(should SUCCEED 2) +PASS : tst_float::float16Comparisons(should SUCCEED 1) +FAIL! : tst_float::float16Comparisons(should FAIL 1) Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 1 + Expected (operandRight): 3 + Loc: [qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)] +FAIL! : tst_float::float16Comparisons(should FAIL 2) Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 0.0001 + Expected (operandRight): 0.0003 + Loc: [qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)] +FAIL! : tst_float::float16Comparisons(should FAIL 3) Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 98 + Expected (operandRight): 99 + Loc: [qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)] +PASS : tst_float::float16Comparisons(should SUCCEED 2) FAIL! : tst_float::compareFloatTests(1e0) Compared floats are not the same (fuzzy compare) Actual (t1): 1 Expected (t3): 3 @@ -28,5 +42,5 @@ FAIL! : tst_float::compareFloatTests(1e+7) Compared floats are not the same (fu Expected (t3): 3e+07 Loc: [qtbase/tests/auto/testlib/selftests/float/tst_float.cpp(0)] PASS : tst_float::cleanupTestCase() -Totals: 4 passed, 6 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 6 passed, 9 failed, 0 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_float ********* diff --git a/tests/auto/testlib/selftests/expected_float.xml b/tests/auto/testlib/selftests/expected_float.xml index 096e1a5b54..1f17d48d1b 100644 --- a/tests/auto/testlib/selftests/expected_float.xml +++ b/tests/auto/testlib/selftests/expected_float.xml @@ -36,6 +36,33 @@ </Incident> <Duration msecs="0"/> </TestFunction> +<TestFunction name="float16Comparisons"> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[should SUCCEED 1]]></DataTag> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> + <DataTag><![CDATA[should FAIL 1]]></DataTag> + <Description><![CDATA[Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 1 + Expected (operandRight): 3]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> + <DataTag><![CDATA[should FAIL 2]]></DataTag> + <Description><![CDATA[Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 0.0001 + Expected (operandRight): 0.0003]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> + <DataTag><![CDATA[should FAIL 3]]></DataTag> + <Description><![CDATA[Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 98 + Expected (operandRight): 99]]></Description> +</Incident> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[should SUCCEED 2]]></DataTag> +</Incident> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="compareFloatTests"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/float/tst_float.cpp" line="0"> <DataTag><![CDATA[1e0]]></DataTag> diff --git a/tests/auto/testlib/selftests/expected_float.xunitxml b/tests/auto/testlib/selftests/expected_float.xunitxml index 5de14e8d9b..87f11ccba5 100644 --- a/tests/auto/testlib/selftests/expected_float.xunitxml +++ b/tests/auto/testlib/selftests/expected_float.xunitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite errors="0" failures="6" tests="4" name="tst_float"> +<testsuite errors="0" failures="9" tests="5" name="tst_float"> <properties> <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/> <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/> @@ -17,6 +17,17 @@ Actual (operandLeft) : 99998 Expected (operandRight): 99999" result="fail"/> </testcase> + <testcase result="fail" name="float16Comparisons"> + <failure tag="should FAIL 1" message="Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 1 + Expected (operandRight): 3" result="fail"/> + <failure tag="should FAIL 2" message="Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 0.0001 + Expected (operandRight): 0.0003" result="fail"/> + <failure tag="should FAIL 3" message="Compared qfloat16s are not the same (fuzzy compare) + Actual (operandLeft) : 98 + Expected (operandRight): 99" result="fail"/> + </testcase> <testcase result="fail" name="compareFloatTests"> <failure tag="1e0" message="Compared floats are not the same (fuzzy compare) Actual (t1): 1 diff --git a/tests/auto/testlib/selftests/float/tst_float.cpp b/tests/auto/testlib/selftests/float/tst_float.cpp index 2fffcc7803..b12bc24c58 100644 --- a/tests/auto/testlib/selftests/float/tst_float.cpp +++ b/tests/auto/testlib/selftests/float/tst_float.cpp @@ -27,6 +27,7 @@ ****************************************************************************/ #include <QtCore/QCoreApplication> +#include <QtCore/qfloat16.h> #include <QtTest/QtTest> #include <QDebug> @@ -36,6 +37,8 @@ class tst_float: public QObject private slots: void floatComparisons() const; void floatComparisons_data() const; + void float16Comparisons() const; + void float16Comparisons_data() const; void compareFloatTests() const; void compareFloatTests_data() const; }; @@ -79,6 +82,42 @@ void tst_float::floatComparisons_data() const << float(100002); } +void tst_float::float16Comparisons() const +{ + QFETCH(qfloat16, operandLeft); + QFETCH(qfloat16, operandRight); + + QCOMPARE(operandLeft, operandRight); +} + +void tst_float::float16Comparisons_data() const +{ + QTest::addColumn<qfloat16>("operandLeft"); + QTest::addColumn<qfloat16>("operandRight"); + + QTest::newRow("should SUCCEED 1") + << qfloat16(0) + << qfloat16(0); + + QTest::newRow("should FAIL 1") + << qfloat16(1.000) + << qfloat16(3.000); + + QTest::newRow("should FAIL 2") + << qfloat16(1.000e-4f) + << qfloat16(3.000e-4f); + + // QCOMPARE for qfloat16s uses qFuzzyCompare() + + QTest::newRow("should FAIL 3") + << qfloat16(98) + << qfloat16(99); + + QTest::newRow("should SUCCEED 2") + << qfloat16(1001) + << qfloat16(1002); +} + void tst_float::compareFloatTests() const { QFETCH(float, t1); diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 1996416e8c..3c5f922c75 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -223,6 +223,7 @@ del re def generateTestData(testname, clean, formats = ('xml', 'txt', 'xunitxml', 'lightxml', 'teamcity', 'tap'), + # Make sure this matches tst_Selftests::runSubTest_data(): extraArgs = { "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2", "benchlibcallgrind": "-callgrind", @@ -236,7 +237,15 @@ def generateTestData(testname, clean, "silent": "-silent", "verbose1": "-v1", "verbose2": "-v2", - }): + }, + # Make sure this matches tst_Selftests::doRunSubTest(): + extraEnv = { + "crashes": { "QTEST_DISABLE_CORE_DUMP": "1", "QTEST_DISABLE_STACK_DUMP": "1" }, + }, + # These are actually *other* crashers, beside those in extraEnv; + # must match tst_Selftests::runSubTest_data(): + crashers = ("assert", "blacklisted", "crashedterminate", + "exceptionthrow", "fetchbogus", "silent")): """Run one test and save its cleaned results. Required arguments are the name of the test directory (the binary @@ -248,6 +257,16 @@ def generateTestData(testname, clean, if not os.path.isfile(path): print("Warning: directory", testname, "contains no test executable") return + env = None + try: + env = extraEnv[testname] + except KeyError: + if env in crashers: + env = extraEnv["crashes"] + if env: + data = os.environ.copy() + data.update(env) + env = data print(" running", testname) for format in formats: @@ -255,7 +274,7 @@ def generateTestData(testname, clean, if testname in extraArgs: cmd += extraArgs[testname].split() - data = subprocess.Popen(cmd, stdout=subprocess.PIPE, + data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env, universal_newlines=True).communicate()[0] with open('expected_' + testname + '.' + format, 'w') as out: out.write('\n'.join(clean(data))) # write() appends a newline, too diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 08c6dec191..f460ca3963 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -520,6 +520,7 @@ void tst_Selftests::runSubTest_data() foreach (QString const& subtest, tests) { QStringList arguments = loggerSet.arguments; + // Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py: if (subtest == "commandlinedata") { arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' '); } @@ -612,6 +613,7 @@ void tst_Selftests::runSubTest_data() if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib")) continue; // Skip benchmark for TeamCity logger + // Keep in sync with generateTestData()'s crashers in generate_expected_output.py: const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow") || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate") || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent") @@ -690,6 +692,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge QProcess proc; QProcessEnvironment environment = processEnvironment(); + // Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py: if (crashes) { environment.insert("QTEST_DISABLE_CORE_DUMP", "1"); environment.insert("QTEST_DISABLE_STACK_DUMP", "1"); @@ -896,16 +899,19 @@ bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, const QString &actualLine, const QString &expectedLine, QString *errorMessage) const { - if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: ")) - && expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) { + if (actualLine == expectedLine) + return true; + + if (subdir == QLatin1String("assert") + && actualLine.contains(QLatin1String("ASSERT: ")) + && expectedLine.contains(QLatin1String("ASSERT: "))) { // Q_ASSERT uses __FILE__, the exact contents of which are // undefined. If have we something that looks like a Q_ASSERT and we // were expecting to see a Q_ASSERT, we'll skip the line. return true; } - if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) - && actualLine != expectedLine) { + if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce"))) { // On some platforms we compile without RTTI, and as a result we never throw an exception if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) { *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine); @@ -944,9 +950,6 @@ bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:"))) return true; - if (actualLine == expectedLine) - return true; - *errorMessage = msgMismatch(actualLine, expectedLine); return false; } diff --git a/tests/auto/testlib/testlib.pro b/tests/auto/testlib/testlib.pro index 25ccc591d6..587c76a189 100644 --- a/tests/auto/testlib/testlib.pro +++ b/tests/auto/testlib/testlib.pro @@ -1,6 +1,7 @@ -TEMPLATE=subdirs -SUBDIRS=\ +TEMPLATE = subdirs +SUBDIRS = \ + outformat \ qsignalspy \ selftests \ -qtHaveModule(widgets):SUBDIRS += qabstractitemmodeltester +qtHaveModule(widgets): SUBDIRS += qabstractitemmodeltester |