diff options
Diffstat (limited to 'tests/auto/testlib')
14 files changed, 178 insertions, 25 deletions
diff --git a/tests/auto/testlib/outformat/outformat.pro b/tests/auto/testlib/outformat/outformat.pro new file mode 100644 index 0000000000..ea02f3167f --- /dev/null +++ b/tests/auto/testlib/outformat/outformat.pro @@ -0,0 +1,7 @@ +CONFIG += testcase +QT = core testlib + +SOURCES += tst_outformat.cpp +TARGET = outformat + +include($$QT_SOURCE_TREE/src/testlib/selfcover.pri) diff --git a/tests/auto/testlib/outformat/tst_outformat.cpp b/tests/auto/testlib/outformat/tst_outformat.cpp new file mode 100644 index 0000000000..5d131159a9 --- /dev/null +++ b/tests/auto/testlib/outformat/tst_outformat.cpp @@ -0,0 +1,71 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include <QtTest/QtTest> + +class tst_OutFormat : public QObject +{ + Q_OBJECT +private slots: + void toHex_data() const; + void toHex() const; + // other formats of interest ? +}; + +void tst_OutFormat::toHex_data() const +{ + QTest::addColumn<QByteArray>("raw"); + QTest::addColumn<QByteArray>("hex"); + + QTest::newRow("empty") << QByteArray("") << QByteArray(""); + QTest::newRow("long") + << QByteArray("Truncates in ellipsis when more than fifty characters long") + << QByteArray("54 72 75 6E 63 61 74 65 73 20 69 6E 20 65 6C 6C " + "69 70 73 69 73 20 77 68 65 6E 20 6D 6F 72 65 20 " + "74 68 61 6E 20 66 69 66 74 79 20 63 68 61 72 61 " + "63 74 ..."); + QTest::newRow("spaces") + << QByteArray(" \t\n\v\f\r") << QByteArray("20 09 0A 0B 0C 0D"); + QTest::newRow("ASCII-escapes") + << QByteArray("\a\b\\\"'\177") << QByteArray("07 08 5C 22 27 7F"); + // These are the ISO Latin-15 , pound, Euro, ..., y-umlaut + QTest::newRow("8-bit-sampler") + << QByteArray("\240\243\244\261\327\360\377") << QByteArray("A0 A3 A4 B1 D7 F0 FF"); +} + +void tst_OutFormat::toHex() const +{ + QFETCH(QByteArray, raw); + QFETCH(QByteArray, hex); + QScopedArrayPointer<char> repr(QTest::toHexRepresentation(raw.constData(), raw.size())); + QCOMPARE(repr.data(), hex); +} + +QTEST_APPLESS_MAIN(tst_OutFormat) + +#include "tst_outformat.moc" diff --git a/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp b/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp index 0593ae74bf..60aa350145 100644 --- a/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp +++ b/tests/auto/testlib/qabstractitemmodeltester/tst_qabstractitemmodeltester.cpp @@ -79,9 +79,9 @@ void tst_QAbstractItemModelTester::treeWidgetModel() root->removeChild(remove); QTreeWidgetItem *parent = new QTreeWidgetItem(&widget, QStringList("parent")); new QTreeWidgetItem(parent, QStringList("child")); - widget.setItemHidden(parent, true); + parent->setHidden(true); - widget.sortByColumn(0); + widget.sortByColumn(0, Qt::AscendingOrder); } void tst_QAbstractItemModelTester::standardItemModel() diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index 36b7699cbd..a923c11416 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -1,12 +1,20 @@ -[pass] +obscure # no such platform; is ignored * + +[pass] +!* + [skip] * + [fail] * -[xpass] -* + [xfail] * + +[xpass] +* + [messages] * diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index 8578752e22..90520385ec 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -45,7 +45,8 @@ private slots: void messages(); }; -// All the tests below have been blacklisted in blacklisted/BLACKLIST +// All the tests below except pass() have been blacklisted in blacklisted/BLACKLIST +// Contrast with ../silent/, for the same tests without blacklisting but with -silent void tst_Blacklisted::pass() { @@ -64,7 +65,7 @@ void tst_Blacklisted::fail() void tst_Blacklisted::xfail() { - QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort); + QEXPECT_FAIL("", "This test should XFAIL then BPASS", Abort); QVERIFY(false); } diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index 4193628e7c..98d7a38ca9 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -25,7 +25,7 @@ </TestFunction> <TestFunction name="xfail"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> - <Description><![CDATA[This test should XFAIL then BFAIL]]></Description> + <Description><![CDATA[This test should XFAIL then BPASS]]></Description> </Incident> <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap new file mode 100644 index 0000000000..f26155ded0 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -0,0 +1,43 @@ +TAP version 13 +# tst_Blacklisted +ok 1 - initTestCase() +ok 2 - pass() # TODO +ok 3 - skip() # SKIP This test should SKIP +not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL) + --- + type: QVERIFY + message: This test should BFAIL + wanted: true (false) + found: false (false) + expected: true (false) + actual: false (false) + at: tst_Blacklisted::fail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:63) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 63 + ... +not ok 5 - xfail() # TODO This test should XFAIL then BPASS + --- + at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:69) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 69 + ... +ok 5 - xfail() # TODO +ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS) +# This is a warning that should not appear in silent test output +# This is an internal testlib warning that should not appear in silent test output +# This is a debug message that should not appear in silent test output +# This is a critical message that should not appear in silent test output +# This is an info message that should not appear in silent test output +# This is an internal testlib info message that should not appear in silent test output +# This is a fatal error message that should still appear in silent test output +not ok 7 - messages() # TODO Received a fatal error. + --- + # Received a fatal error. + at: tst_Blacklisted::messages() (Unknown file:0) + file: Unknown file + line: 0 + ... +1..7 +# tests 7 +# pass 1 +# fail 1 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index 8180a7ce76..df58208fb3 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -7,7 +7,7 @@ ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='fail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted'] -##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BPASS' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted'] ##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index 6fa2403b59..3cfe40eb12 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -6,7 +6,7 @@ SKIP : tst_Blacklisted::skip() This test should SKIP Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BFAIL +XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BPASS Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BPASS : tst_Blacklisted::xfail() XPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS) diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index 443bc6b199..f0387bae00 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -27,7 +27,7 @@ </TestFunction> <TestFunction name="xfail"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> - <Description><![CDATA[This test should XFAIL then BFAIL]]></Description> + <Description><![CDATA[This test should XFAIL then BPASS]]></Description> </Incident> <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml index 2752bc18b4..e2d0cd009b 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml @@ -12,7 +12,7 @@ </testcase> <testcase result="bfail" name="fail"/> <testcase result="xfail" name="xfail"> - <!-- message="This test should XFAIL then BFAIL" type="info" --> + <!-- message="This test should XFAIL then BPASS" type="info" --> </testcase> <testcase result="xpass" name="xpass"> <failure message="'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/> @@ -28,7 +28,7 @@ </testcase> <system-err> <![CDATA[This test should SKIP]]> -<![CDATA[This test should XFAIL then BFAIL]]> +<![CDATA[This test should XFAIL then BPASS]]> <![CDATA[This is a warning that should not appear in silent test output]]> <![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a debug message that should not appear in silent test output]]> diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 1996416e8c..3c5f922c75 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -223,6 +223,7 @@ del re def generateTestData(testname, clean, formats = ('xml', 'txt', 'xunitxml', 'lightxml', 'teamcity', 'tap'), + # Make sure this matches tst_Selftests::runSubTest_data(): extraArgs = { "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2", "benchlibcallgrind": "-callgrind", @@ -236,7 +237,15 @@ def generateTestData(testname, clean, "silent": "-silent", "verbose1": "-v1", "verbose2": "-v2", - }): + }, + # Make sure this matches tst_Selftests::doRunSubTest(): + extraEnv = { + "crashes": { "QTEST_DISABLE_CORE_DUMP": "1", "QTEST_DISABLE_STACK_DUMP": "1" }, + }, + # These are actually *other* crashers, beside those in extraEnv; + # must match tst_Selftests::runSubTest_data(): + crashers = ("assert", "blacklisted", "crashedterminate", + "exceptionthrow", "fetchbogus", "silent")): """Run one test and save its cleaned results. Required arguments are the name of the test directory (the binary @@ -248,6 +257,16 @@ def generateTestData(testname, clean, if not os.path.isfile(path): print("Warning: directory", testname, "contains no test executable") return + env = None + try: + env = extraEnv[testname] + except KeyError: + if env in crashers: + env = extraEnv["crashes"] + if env: + data = os.environ.copy() + data.update(env) + env = data print(" running", testname) for format in formats: @@ -255,7 +274,7 @@ def generateTestData(testname, clean, if testname in extraArgs: cmd += extraArgs[testname].split() - data = subprocess.Popen(cmd, stdout=subprocess.PIPE, + data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env, universal_newlines=True).communicate()[0] with open('expected_' + testname + '.' + format, 'w') as out: out.write('\n'.join(clean(data))) # write() appends a newline, too diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index c5f847562e..008cce6218 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -520,6 +520,7 @@ void tst_Selftests::runSubTest_data() foreach (QString const& subtest, tests) { QStringList arguments = loggerSet.arguments; + // Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py: if (subtest == "commandlinedata") { arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' '); } @@ -612,6 +613,7 @@ void tst_Selftests::runSubTest_data() if (loggerSet.name.contains("teamcity") && subtest.startsWith("benchlib")) continue; // Skip benchmark for TeamCity logger + // Keep in sync with generateTestData()'s crashers in generate_expected_output.py: const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow") || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate") || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent") @@ -687,6 +689,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge QProcess proc; QProcessEnvironment environment = processEnvironment(); + // Keep in sync with generateTestData()'s extraEnv in generate_expected_output.py: if (crashes) { environment.insert("QTEST_DISABLE_CORE_DUMP", "1"); environment.insert("QTEST_DISABLE_STACK_DUMP", "1"); @@ -893,16 +896,19 @@ bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, const QString &actualLine, const QString &expectedLine, QString *errorMessage) const { - if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: ")) - && expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) { + if (actualLine == expectedLine) + return true; + + if (subdir == QLatin1String("assert") + && actualLine.contains(QLatin1String("ASSERT: ")) + && expectedLine.contains(QLatin1String("ASSERT: "))) { // Q_ASSERT uses __FILE__, the exact contents of which are // undefined. If have we something that looks like a Q_ASSERT and we // were expecting to see a Q_ASSERT, we'll skip the line. return true; } - if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) - && actualLine != expectedLine) { + if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce"))) { // On some platforms we compile without RTTI, and as a result we never throw an exception if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) { *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine); @@ -941,9 +947,6 @@ bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:"))) return true; - if (actualLine == expectedLine) - return true; - *errorMessage = msgMismatch(actualLine, expectedLine); return false; } diff --git a/tests/auto/testlib/testlib.pro b/tests/auto/testlib/testlib.pro index 25ccc591d6..587c76a189 100644 --- a/tests/auto/testlib/testlib.pro +++ b/tests/auto/testlib/testlib.pro @@ -1,6 +1,7 @@ -TEMPLATE=subdirs -SUBDIRS=\ +TEMPLATE = subdirs +SUBDIRS = \ + outformat \ qsignalspy \ selftests \ -qtHaveModule(widgets):SUBDIRS += qabstractitemmodeltester +qtHaveModule(widgets): SUBDIRS += qabstractitemmodeltester |