diff options
author | Leander Beernaert <leander.beernaert@qt.io> | 2020-01-22 13:47:08 +0100 |
---|---|---|
committer | Leander Beernaert <leander.beernaert@qt.io> | 2020-01-24 13:17:33 +0100 |
commit | 502d3d6744913899da87acfda5ebdab42c40329e (patch) | |
tree | 16658a328503bfd5a62b4fd5d69ffb66e9854b18 /src/testlib | |
parent | d1be8b9ceb2c7b20bbe53a07154c79699540ea3d (diff) | |
parent | 06bb315beb6c2c398223cfe52cbc7f66e14a8557 (diff) |
Merge remote-tracking branch 'origin/dev' into merge-dev
Change-Id: I31b761cfd5ea01373c60d02a5da8c33398d34739
Diffstat (limited to 'src/testlib')
-rw-r--r-- | src/testlib/doc/qttestlib.qdocconf | 2 | ||||
-rw-r--r-- | src/testlib/doc/snippets/code/doc_src_cmakelists.txt | 14 | ||||
-rw-r--r-- | src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp | 29 | ||||
-rw-r--r-- | src/testlib/doc/src/qt-webpages.qdoc | 26 | ||||
-rw-r--r-- | src/testlib/doc/src/qttest-best-practices.qdoc | 533 | ||||
-rw-r--r-- | src/testlib/doc/src/qttest-index.qdoc | 3 | ||||
-rw-r--r-- | src/testlib/doc/src/qttestlib-manual.qdoc | 84 | ||||
-rw-r--r-- | src/testlib/qabstractitemmodeltester.cpp | 2 | ||||
-rw-r--r-- | src/testlib/qsignalspy.qdoc | 8 | ||||
-rw-r--r-- | src/testlib/qtest.h | 2 | ||||
-rw-r--r-- | src/testlib/qtestcase.qdoc | 167 |
11 files changed, 811 insertions, 59 deletions
diff --git a/src/testlib/doc/qttestlib.qdocconf b/src/testlib/doc/qttestlib.qdocconf index 73310221cf..52e1480295 100644 --- a/src/testlib/doc/qttestlib.qdocconf +++ b/src/testlib/doc/qttestlib.qdocconf @@ -28,7 +28,7 @@ qhp.QtTestLib.subprojects.classes.sortPages = true tagfile = ../../../doc/qttestlib/qttestlib.tags -depends += qtcore qtdoc qtwidgets qtgui qmake qtqmltest +depends += qtcore qtdoc qtwidgets qtgui qmake qtqmltest qtcmake headerdirs += .. diff --git a/src/testlib/doc/snippets/code/doc_src_cmakelists.txt b/src/testlib/doc/snippets/code/doc_src_cmakelists.txt new file mode 100644 index 0000000000..96dbe1acee --- /dev/null +++ b/src/testlib/doc/snippets/code/doc_src_cmakelists.txt @@ -0,0 +1,14 @@ +project(mytest LANGUAGES CXX) + +find_package(Qt5Test REQUIRED) + +set(CMAKE_INCLUDE_CURRENT_DIR ON) + +set(CMAKE_AUTOMOC ON) + +enable_testing(true) + +add_executable(mytest tst_mytest.cpp) +add_test(NAME mytest COMMAND mytest) + +target_link_libraries(mytest PRIVATE Qt5::Test) diff --git a/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp b/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp index 5f71828595..2dc4fe49ef 100644 --- a/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp +++ b/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp @@ -53,12 +53,12 @@ void wrapInFunction() { //! [0] -QVERIFY(1 + 1 == 2); + QVERIFY(spy.isValid()) //! [0] //! [1] -QVERIFY2(1 + 1 == 2, "A breach in basic arithmetic occurred."); +QVERIFY2(qIsNaN(0.0 / 0.0), "Ill-defined division produced unambiguous result."); //! [1] @@ -324,3 +324,28 @@ void TestQLocale::roundTripInt() QVERIFY(ok); } //! [31] + + +//! [32] +bool opened = file.open(QIODevice::WriteOnly); +QVERIFY(opened); +//! [32] + + +//! [33] +QVERIFY2(file.open(QIODevice::WriteOnly), + qPrintable(QString("open %1: %2").arg(file.fileName()).arg(file.errorString())); +//! [33] + +//! [34] +QT_BEGIN_NAMESPACE +namespace QTest { + template <> char *toString<MyType>(const MyType &t) + { + char *repr = new char[t.reprSize()]; + t.writeRepr(repr); + return repr; + } +} +QT_END_NAMESPACE +//! [34] diff --git a/src/testlib/doc/src/qt-webpages.qdoc b/src/testlib/doc/src/qt-webpages.qdoc index 29a2589a4c..976435e668 100644 --- a/src/testlib/doc/src/qt-webpages.qdoc +++ b/src/testlib/doc/src/qt-webpages.qdoc @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2019 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the documentation of the Qt Toolkit. @@ -25,10 +25,26 @@ ** ****************************************************************************/ /*! - \externalpage http://blog.qt.io - \title Qt Labs + \externalpage https://blog.qt.io/blog/2008/12/05/qtestlib-now-with-nice-graphs-pointing-upwards/ + \title qtestlib-tools Announcement */ + /*! - \externalpage http://blog.qt.io/blog/2008/12/05/qtestlib-now-with-nice-graphs-pointing-upwards/ - \title qtestlib-tools Announcement + \externalpage https://www.froglogic.com/coco/ + \title Froglogic Coco Code Coverage +*/ + +/*! + \externalpage https://gcc.gnu.org/onlinedocs/gcc/Gcov.html + \title gcov +*/ + +/*! + \externalpage https://github.com/google/googletest/tree/master/googlemock + \title Googletest Mocking (gMock) Framework +*/ + +/*! + \externalpage https://www.itk.org/Wiki/CMake_Testing_With_CTest + \title CMake/Testing With CTest */ diff --git a/src/testlib/doc/src/qttest-best-practices.qdoc b/src/testlib/doc/src/qttest-best-practices.qdoc new file mode 100644 index 0000000000..7143e644fd --- /dev/null +++ b/src/testlib/doc/src/qttest-best-practices.qdoc @@ -0,0 +1,533 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:FDL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qttest-best-practices.qdoc + + \title Qt Test Best Practices + + \brief Guidelines for creating Qt tests. + + We recommend that you add Qt tests for bug fixes and new features. Before + you try to fix a bug, add a \e {regression test} (ideally automatic) that + fails before the fix, exhibiting the bug, and passes after the fix. While + you're developing new features, add tests to verify that they work as + intended. + + Conforming to a set of coding standards will make it more likely for + Qt autotests to work reliably in all environments. For example, some + tests need to read data from disk. If no standards are set for how this + is done, some tests won't be portable. For example, a test that assumes + its test-data files are in the current working directory only works for + an in-source build. In a shadow build (outside the source directory), the + test will fail to find its data. + + The following sections contain guidelines for writing Qt tests: + + \list + \li \l {General Principles} + \li \l {Writing Reliable Tests} + \li \l {Improving Test Output} + \li \l {Writing Testable Code} + \li \l {Setting up Test Machines} + \endlist + + \section1 General Principles + + The following sections provide general guidelines for writing unit tests: + + \list + \li \l {Verify Tests} + \li \l {Give Test Functions Descriptive Names} + \li \l {Write Self-contained Test Functions} + \li \l {Test the Full Stack} + \li \l {Make Tests Complete Quickly} + \li \l {Use Data-driven Testing} + \li \l {Use Coverage Tools} + \li \l {Select Appropriate Mechanisms to Exclude Tests} + \li \l {Avoid Q_ASSERT} + \endlist + + \section2 Verify Tests + + Write and commit your tests along with your fix or new feature on a new + branch. Once you're done, you can check out the branch on which your work + is based, and then check out into this branch the test-files for your new + tests. This enables you to verify that the tests do fail on the prior + branch, and therefore actually do catch a bug or test a new feature. + + For example, the workflow to fix a bug in the \c QDateTime class could be + like this if you use the Git version control system: + + \list 1 + \li Create a branch for your fix and test: + \c {git checkout -b fix-branch 5.14} + \li Write a test and fix the bug. + \li Build and test with both the fix and the new test, to verify that + the new test passes with the fix. + \li Add the fix and test to your branch: + \c {git add tests/auto/corelib/time/qdatetime/tst_qdatetime.cpp src/corelib/time/qdatetime.cpp} + \li Commit the fix and test to your branch: + \c {git commit -m 'Fix bug in QDateTime'} + \li To verify that the test actually catches something for which you + needed the fix, checkout the branch you based your own branch on: + \c {git checkout 5.14} + \li Checkout only the test file to the 5.14 branch: + \c {git checkout fix-branch -- tests/auto/corelib/time/qdatetime/tst_qdatetime.cpp} + + Only the test is now on the fix-branch. The rest of the source tree + is still on 5.14. + \li Build and run the test to verify that it fails on 5.14, and + therefore does indeed catch a bug. + \li You can now return to the fix branch: + \c {git checkout fix-branch} + \li Alternatively, you can restore your work tree to a clean state on + 5.14: + \c{git checkout HEAD -- tests/auto/corelib/time/qdatetime/tst_qdatetime.cpp} + \endlist + + When you're reviewing a change, you can adapt this workflow to check that + the change does indeed come with a test for a problem it does fix. + + \section2 Give Test Functions Descriptive Names + + Naming test cases is important. The test name appears in the failure report + for a test run. For data-driven tests, the name of the data row also appears + in the failure report. The names give those reading the report a first + indication of what has gone wrong. + + Test function names should make it obvious what the function is trying to + test. Do not simply use the bug-tracking identifier, because the identifiers + become obsolete if the bug-tracker is replaced. Also, some bug-trackers may + not be accessible to all users. When the bug report may be of interest to + later readers of the test code, you can mention it in a comment alongside a + relevant part of the test. + + Likewise, when writing data-driven tests, give descriptive names to the + test-cases, that indicate what aspect of the functionality each focuses on. + Do not simply number the test-case, or use bug-tracking identifiers. Someone + reading the test output will have no idea what the numbers or identifiers + mean. You can add a comment on the test-row that mentions the bug-tracking + identifier, when relevant. + + \section2 Write Self-contained Test Functions + + Within a test program, test functions should be independent of each other + and they should not rely upon previous test functions having been run. You + can check this by running the test function on its own with + \c {tst_foo testname}. + + Do not re-use instances of the class under test in several tests. Test + instances (for example widgets) should not be member variables of the + tests, but preferably be instantiated on the stack to ensure proper + cleanup even if a test fails, so that tests do not interfere with + each other. + + \section2 Test the Full Stack + + If an API is implemented in terms of pluggable or platform-specific backends + that do the heavy-lifting, make sure to write tests that cover the + code-paths all the way down into the backends. Testing the upper layer API + parts using a mock backend is a nice way to isolate errors in the API layer + from the backends, but it is complementary to tests that run the actual + implementation with real-world data. + + \section2 Make Tests Complete Quickly + + Tests should not waste time by being unnecessarily repetitious, by using + inappropriately large volumes of test data, or by introducing needless + idle time. + + This is particularly true for unit testing, where every second of extra + unit test execution time makes CI testing of a branch across multiple + targets take longer. Remember that unit testing is separate from load and + reliability testing, where larger volumes of test data and longer test + runs are expected. + + Benchmark tests, which typically execute the same test multiple times, + should be located in a separate \c tests/benchmarks directory and they + should not be mixed with functional unit tests. + + \section2 Use Data-driven Testing + + Data-driven tests make it easier to add new tests for boundary conditions + found in later bug reports. + + Using a data-driven test rather than testing several items in sequence in + a test saves repetition of very similar code and ensures later cases are + tested even when earlier ones fail. It also encourages systematic and + uniform testing, because the same tests are applied to each data sample. + + \section2 Use Coverage Tools + + Use a coverage tool such as \l {Froglogic Coco Code Coverage} or \l {gcov} + to help write tests that cover as many statements, branches, and conditions + as possible in the function or class being tested. The earlier this is done + in the development cycle for a new feature, the easier it will be to catch + regressions later when the code is refactored. + + \section2 Select Appropriate Mechanisms to Exclude Tests + + It is important to select the appropriate mechanism to exclude inapplicable + tests: \l QSKIP(), using conditional statements to exclude parts of a test + function, or not building the test for a particular platform. + + Use QSKIP() to handle cases where a whole test function is found at run-time + to be inapplicable in the current test environment. When just a part of a + test function is to be skipped, a conditional statement can be used, + optionally with a \c qDebug() call to report the reason for skipping the + inapplicable part. + + Test functions or data rows of a data-driven test can be limited to + particular platforms, or to particular features being enabled using + \c{#if}. However, beware of \l moc limitations when using \c{#if} to + skip test functions. The \c moc preprocessor does not have access to + all the \c builtin macros of the compiler that are often used for + feature detection of the compiler. Therefore, \c moc might get a different + result for a preprocessor condition from that seen by the rest of your + code. This may result in \c moc generating meta-data for a test slot that + the actual compiler skips, or omitting the meta-data for a test slot that + is actually compiled into the class. In the first case, the test will + attempt to run a slot that is not implemented. In the second case, the + test will not attempt to run a test slot even though it should. + + If an entire test program is inapplicable for a specific platform or + unless a particular feature is enabled, the best approach is to use the + parent directory's \c .pro file to avoid building the test. For example, + if the \c tests/auto/gui/someclass test is not valid for \macOS, add the + following line to \c tests/auto/gui.pro: + + \badcode + mac*: SUBDIRS -= someclass + \endcode + + \section2 Avoid Q_ASSERT + + The \l Q_ASSERT macro causes a program to abort whenever the asserted + condition is \c false, but only if the software was built in debug mode. + In both release and debug-and-release builds, \c Q_ASSERT does nothing. + + \c Q_ASSERT should be avoided because it makes tests behave differently + depending on whether a debug build is being tested, and because it causes + a test to abort immediately, skipping all remaining test functions and + returning incomplete or malformed test results. + + It also skips any tear-down or tidy-up that was supposed to happen at the + end of the test, and might therefore leave the workspace in an untidy state, + which might cause complications for further tests. + + Instead of \c Q_ASSERT, the \l QCOMPARE() or \l QVERIFY() macro variants + should be used. They cause the current test to report a failure and + terminate, but allow the remaining test functions to be executed and the + entire test program to terminate normally. \l QVERIFY2() even allows a + descriptive error message to be recorded in the test log. + + \section1 Writing Reliable Tests + + The following sections provide guidelines for writing reliable tests: + + \list + \li \l {Avoid Side-effects in Verification Steps} + \li \l {Avoid Fixed Timeouts} + \li \l {Beware of Timing-dependent Behavior} + \li \l {Avoid Bitmap Capture and Comparison} + \endlist + + \section2 Avoid Side-effects in Verification Steps + + When performing verification steps in an autotest using \l QCOMPARE(), + \l QVERIFY(), and so on, side-effects should be avoided. Side-effects + in verification steps can make a test difficult to understand. Also, + they can easily break a test in ways that are difficult to diagnose + when the test is changed to use \l QTRY_VERIFY(), \l QTRY_COMPARE() or + \l QBENCHMARK(). These can execute the passed expression multiple times, + thus repeating any side-effects. + + When side-effects are unavoidable, ensure that the prior state is restored + at the end of the test function, even if the test fails. This commonly + requires use of an RAII (resource acquisition is initialization) class + that restores state when the function returns, or a \c cleanup() method. + Do not simply put the restoration code at the end of the test. If part of + the test fails, such code will be skipped and the prior state will not be + restored. + + \section2 Avoid Fixed Timeouts + + Avoid using hard-coded timeouts, such as QTest::qWait() to wait for some + conditions to become true. Consider using the \l QSignalSpy class, + the \l QTRY_VERIFY() or \l QTRY_COMPARE() macros, or the \c QSignalSpy + class in conjunction with the \c QTRY_ macro variants. + + The \c qWait() function can be used to set a delay for a fixed period + between performing some action and waiting for some asynchronous behavior + triggered by that action to be completed. For example, changing the state + of a widget and then waiting for the widget to be repainted. However, + such timeouts often cause failures when a test written on a workstation is + executed on a device, where the expected behavior might take longer to + complete. Increasing the fixed timeout to a value several times larger + than needed on the slowest test platform is not a good solution, because + it slows down the test run on all platforms, particularly for table-driven + tests. + + If the code under test issues Qt signals on completion of the asynchronous + behavior, a better approach is to use the \l QSignalSpy class to notify + the test function that the verification step can now be performed. + + If there are no Qt signals, use the \c QTRY_COMPARE() and \c QTRY_VERIFY() + macros, which periodically test a specified condition until it becomes true + or some maximum timeout is reached. These macros prevent the test from + taking longer than necessary, while avoiding breakages when tests are + written on workstations and later executed on embedded platforms. + + If there are no Qt signals, and you are writing the test as part of + developing a new API, consider whether the API could benefit from the + addition of a signal that reports the completion of the asynchronous + behavior. + + \section2 Beware of Timing-dependent Behavior + + Some test strategies are vulnerable to timing-dependent behavior of certain + classes, which can lead to tests that fail only on certain platforms or that + do not return consistent results. + + One example of this is text-entry widgets, which often have a blinking + cursor that can make comparisons of captured bitmaps succeed or fail + depending on the state of the cursor when the bitmap is captured. This, + in turn, may depend on the speed of the machine executing the test. + + When testing classes that change their state based on timer events, the + timer-based behavior needs to be taken into account when performing + verification steps. Due to the variety of timing-dependent behavior, there + is no single generic solution to this testing problem. + + For text-entry widgets, potential solutions include disabling the cursor + blinking behavior (if the API provides that feature), waiting for the + cursor to be in a known state before capturing a bitmap (for example, by + subscribing to an appropriate signal if the API provides one), or + excluding the area containing the cursor from the bitmap comparison. + + \section2 Avoid Bitmap Capture and Comparison + + While verifying test results by capturing and comparing bitmaps is sometimes + necessary, it can be quite fragile and labor-intensive. + + For example, a particular widget may have different appearance on different + platforms or with different widget styles, so reference bitmaps may need to + be created multiple times and then maintained in the future as Qt's set of + supported platforms evolves. Making changes that affect the bitmap thus + means having to recreate the expected bitmaps on each supported platform, + which would require access to each platform. + + Bitmap comparisons can also be influenced by factors such as the test + machine's screen resolution, bit depth, active theme, color scheme, + widget style, active locale (currency symbols, text direction, and so + on), font size, transparency effects, and choice of window manager. + + Where possible, use programmatic means, such as verifying properties of + objects and variables, instead of capturing and comparing bitmaps. + + \section1 Improving Test Output + + The following sections provide guidelines for producing readable and + helpful test output: + + \list + \li \l {Explicitly Ignore Expected Warnings} + \li \l {Avoid Printing Debug Messages from Autotests} + \li \l {Write Well-structured Diagnostic Code} + \endlist + + \section2 Explicitly Ignore Expected Warnings + + If a test is expected to cause Qt to output a warning or debug message + on the console, it should call \l QTest::ignoreMessage() to filter that + message out of the test output and to fail the test if the message is + not output. + + If such a message is only output when Qt is built in debug mode, use + \l QLibraryInfo::isDebugBuild() to determine whether the Qt libraries + were built in debug mode. Using \c{#ifdef QT_DEBUG} is not enough, as + it will only tell you whether the test was built in debug mode, and + that does not guarantee that the Qt libraries were also built in debug + mode. + + \section2 Avoid Printing Debug Messages from Autotests + + Autotests should not produce any unhandled warning or debug messages. + This will allow the CI Gate to treat new warning or debug messages as + test failures. + + Adding debug messages during development is fine, but these should be + either disabled or removed before a test is checked in. + + \section2 Write Well-structured Diagnostic Code + + Any diagnostic output that would be useful if a test fails should be part + of the regular test output rather than being commented-out, disabled by + preprocessor directives, or enabled only in debug builds. If a test fails + during continuous integration, having all of the relevant diagnostic output + in the CI logs could save you a lot of time compared to enabling the + diagnostic code and testing again. Epecially, if the failure was on a + platform that you don't have on your desktop. + + Diagnostic messages in tests should use Qt's output mechanisms, such as + \c qDebug() and \c qWarning(), rather than \c stdio.h or \c iostream.h output + mechanisms. The latter bypass Qt's message handling and prevent the + \c -silent command-line option from suppressing the diagnostic messages. + This could result in important failure messages being hidden in a large + volume of debugging output. + + \section1 Writing Testable Code + + The following sections provide guidelines for writing code that is easy to + test: + + \list + \li \l {Break Dependencies} + \li \l {Compile All Classes into Libraries} + \endlist + + \section2 Break Dependencies + + The idea of unit testing is to use every class in isolation. Since many + classes instantiate other classes, it is not possible to instantiate one + class separately. Therefore, you should use a technique called + \e {dependency injection} that separates object creation from object use. + A factory is responsible for building object trees. Other objects manipulate + these objects through abstract interfaces. + + This technique works well for data-driven applications. For GUI + applications, this approach can be difficult as objects are frequently + created and destructed. To verify the correct behavior of classes that + depend on abstract interfaces, \e mocking can be used. For example, see + \l {Googletest Mocking (gMock) Framework}. + + \section2 Compile All Classes into Libraries + + In small to medium sized projects, a build script typically lists all + source files and then compiles the executable in one go. This means that + the build scripts for the tests must list the needed source files again. + + It is easier to list the source files and the headers only once in a + script to build a static library. Then the \c main() function will be + linked against the static library to build the executable and the tests + will be linked against the static libraries. + + For projects where the same source files are used in building several + programs, it may be more appropriate to build the shared classes into + a dynamically-linked (or shared object) library that each program, + including the test programs, can load at run-time. Again, having the + compiled code in a library helps to avoid duplication in the description + of which components to combine to make the various programs. + + \section1 Setting up Test Machines + + The following sections discuss common problems caused by test machine setup: + + \list + \li \l {Screen Savers} + \li \l {System Dialogs} + \li \l {Display Usage} + \li \l {Window Managers} + \endlist + + All of these problems can typically be solved by the judicious use of + virtualisation. + + \section2 Screen Savers + + Screen savers can interfere with some of the tests for GUI classes, causing + unreliable test results. Screen savers should be disabled to ensure that + test results are consistent and reliable. + + \section2 System Dialogs + + Dialogs displayed unexpectedly by the operating system or other running + applications can steal input focus from widgets involved in an autotest, + causing unreproducible failures. + + Examples of typical problems include online update notification dialogs + on macOS, false alarms from virus scanners, scheduled tasks such as virus + signature updates, software updates pushed out to workstations, and chat + programs popping up windows on top of the stack. + + \section2 Display Usage + + Some tests use the test machine's display, mouse, and keyboard, and can + thus fail if the machine is being used for something else at the same + time or if multiple tests are run in parallel. + + The CI system uses dedicated test machines to avoid this problem, but if + you don't have a dedicated test machine, you may be able to solve this + problem by running the tests on a second display. + + On Unix, one can also run the tests on a nested or virtual X-server, such as + Xephyr. For example, to run the entire set of tests on Xephyr, execute the + following commands: + + \code + Xephyr :1 -ac -screen 1920x1200 >/dev/null 2>&1 & + sleep 5 + DISPLAY=:1 icewm >/dev/null 2>&1 & + cd tests/auto + make + DISPLAY=:1 make -k -j1 check + \endcode + + Users of NVIDIA binary drivers should note that Xephyr might not be able to + provide GLX extensions. Forcing Mesa libGL might help: + + \code + export LD_PRELOAD=/usr/lib/mesa-diverted/x86_64-linux-gnu/libGL.so.1 + \endcode + + However, when tests are run on Xephyr and the real X-server with different + libGL versions, the QML disk cache can make the tests crash. To avoid this, + use \c QML_DISABLE_DISK_CACHE=1. + + Alternatively, use the offscreen plugin: + + \code + TESTARGS="-platform offscreen" make check -k -j1 + \endcode + + \section2 Window Managers + + On Unix, at least two autotests (\c tst_examples and \c tst_gestures) + require a window manager to be running. Therefore, if running these + tests under a nested X-server, you must also run a window manager + in that X-server. + + Your window manager must be configured to position all windows on the + display automatically. Some windows managers, such as Tab Window Manager + (twm), have a mode for manually positioning new windows, and this prevents + the test suite from running without user interaction. + + \note Tab Window Manager is not suitable for running the full suite of + Qt autotests, as the \c tst_gestures autotest causes it to forget its + configuration and revert to manual window placement. +*/ diff --git a/src/testlib/doc/src/qttest-index.qdoc b/src/testlib/doc/src/qttest-index.qdoc index 23be46b431..e31f232069 100644 --- a/src/testlib/doc/src/qttest-index.qdoc +++ b/src/testlib/doc/src/qttest-index.qdoc @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2019 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the documentation of the Qt Toolkit. @@ -56,6 +56,7 @@ \list \li \l{Qt Test Overview} + \li \l{Qt Test Best Practices} \li \l{Qt Test Tutorial} \endlist diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc index bb379fe029..89edabf3f3 100644 --- a/src/testlib/doc/src/qttestlib-manual.qdoc +++ b/src/testlib/doc/src/qttestlib-manual.qdoc @@ -83,6 +83,10 @@ \li Custom types can easily be added to the test data and test output. \endtable + You can use a Qt Creator wizard to create a project that contains Qt tests + and build and run them directly from Qt Creator. For more information, see + \l {Qt Creator: Running Autotests}{Running Autotests}. + \section1 Creating a Test To create a test, subclass QObject and add one or more private slots to it. Each @@ -102,6 +106,20 @@ \li \c{cleanup()} will be called after every test function. \endlist + Use \c initTestCase() for preparing the test. Every test should leave the + system in a usable state, so it can be run repeatedly. Cleanup operations + should be handled in \c cleanupTestCase(), so they get run even if the test + fails. + + Use \c init() for preparing a test function. Every test function should + leave the system in a usable state, so it can be run repeatedly. Cleanup + operations should be handled in \c cleanup(), so they get run even if the + test function fails and exits early. + + Alternatively, you can use RAII (resource acquisition is initialization), + with cleanup operations called in destructors, to ensure they happen when + the test function returns and the object moves out of scope. + If \c{initTestCase()} fails, no test function will be executed. If \c{init()} fails, the following test function will not be executed, the test will proceed to the next test function. @@ -119,6 +137,41 @@ \if !defined(qtforpython) \section1 Building a Test + You can build an executable that contains one test class that typically + tests one class of production code. However, usually you would want to + test several classes in a project by running one command. + + See \l {Chapter 1: Writing a Unit Test}{Writing a Unit Test} for a step by + step explanation. + + \section2 Building with CMake and CTest + + You can use \l {Building with CMake and CTest} to create a test. + \l{https://cmake.org/cmake/help/latest/manual/ctest.1.html}{CTest} enables + you to include or exclude tests based on a regular expression that is + matched against the test name. You can further apply the \c LABELS property + to a test and CTest can then include or exclude tests based on those labels. + All labeled targets will be run when \c {test} target is called on the + command line. + + There are several other advantages with CMake. For example, the result of + a test run can be published on a web server using CDash with virtually no + effort. + + CTest scales to very different unit test frameworks, and works out of the + box with QTest. + + The following is an example of a CMakeLists.txt file that specifies the + project name and the language used (here, \e mytest and C++), the Qt + modules required for building the test (Qt5Test), and the files that are + included in the test (\e tst_mytest.cpp). + + \quotefile code/doc_src_cmakelists.txt + + For more information about the options you have, see \l {Build with CMake}. + + \section2 Building with qmake + If you are using \c qmake as your build tool, just add the following to your project file: @@ -132,14 +185,14 @@ See the \l{Building a Testcase}{qmake manual} for more information about \c{make check}. + \section2 Building with Other Tools + If you are using other build tools, make sure that you add the location of the Qt Test header files to your include path (usually \c{include/QtTest} under your Qt installation directory). If you are using a release build of Qt, link your test to the \c QtTest library. For debug builds, use \c{QtTest_debug}. - See \l {Chapter 1: Writing a Unit Test}{Writing a Unit Test} for a step by - step explanation. \endif \section1 Qt Test Command Line Arguments @@ -308,10 +361,35 @@ \section1 Creating a Benchmark To create a benchmark, follow the instructions for creating a test and then add a - QBENCHMARK macro to the test function that you want to benchmark. + \l QBENCHMARK macro or \l QTest::setBenchmarkResult() to the test function that + you want to benchmark. In the following code snippet, the macro is used: \snippet code/doc_src_qtestlib.cpp 12 + A test function that measures performance should contain either a single + \c QBENCHMARK macro or a single call to \c setBenchmarkResult(). Multiple + occurrences make no sense, because only one performance result can be + reported per test function, or per data tag in a data-driven setup. + + Avoid changing the test code that forms (or influences) the body of a + \c QBENCHMARK macro, or the test code that computes the value passed to + \c setBenchmarkResult(). Differences in successive performance results + should ideally be caused only by changes to the product you are testing. + Changes to the test code can potentially result in misleading report of + a change in performance. If you do need to change the test code, make + that clear in the commit message. + + In a performance test function, the \c QBENCHMARK or \c setBenchmarkResult() + should be followed by a verification step using \l QCOMPARE(), \l QVERIFY(), + and so on. You can then flag a performance result as \e invalid if another + code path than the intended one was measured. A performance analysis tool + can use this information to filter out invalid results. + For example, an unexpected error condition will typically cause the program + to bail out prematurely from the normal program execution, and thus falsely + show a dramatic performance increase. + + \section2 Selecting the Measurement Back-end + The code inside the QBENCHMARK macro will be measured, and possibly also repeated several times in order to get an accurate measurement. This depends on the selected measurement back-end. Several back-ends are available. They can be selected on the diff --git a/src/testlib/qabstractitemmodeltester.cpp b/src/testlib/qabstractitemmodeltester.cpp index 859966c0e3..2e05097122 100644 --- a/src/testlib/qabstractitemmodeltester.cpp +++ b/src/testlib/qabstractitemmodeltester.cpp @@ -605,7 +605,7 @@ void QAbstractItemModelTesterPrivate::data() // Check that the alignment is one we know about QVariant textAlignmentVariant = model->data(model->index(0, 0), Qt::TextAlignmentRole); if (textAlignmentVariant.isValid()) { - Qt::Alignment alignment = textAlignmentVariant.value<Qt::Alignment>(); + Qt::Alignment alignment = qvariant_cast<Qt::Alignment>(textAlignmentVariant); MODELTESTER_COMPARE(alignment, (alignment & (Qt::AlignHorizontal_Mask | Qt::AlignVertical_Mask))); } diff --git a/src/testlib/qsignalspy.qdoc b/src/testlib/qsignalspy.qdoc index d532ad478d..42e02ed601 100644 --- a/src/testlib/qsignalspy.qdoc +++ b/src/testlib/qsignalspy.qdoc @@ -98,13 +98,13 @@ \snippet code/doc_src_qsignalspy.cpp 6 */ -/*! \fn QSignalSpy(const QObject *obj, const QMetaMethod &signal) +/*! \fn QSignalSpy::QSignalSpy(const QObject *obj, const QMetaMethod &signal) \since 5.14 Constructs a new QSignalSpy that listens for emissions of the \a signal - from the QObject \a object. If QSignalSpy is not able to listen for a - valid signal (for example, because \a object is \nullptr or \a signal does - not denote a valid signal of \a object), an explanatory warning message + from the QObject \a obj. If QSignalSpy is not able to listen for a + valid signal (for example, because \a obj is \nullptr or \a signal does + not denote a valid signal of \a obj), an explanatory warning message will be output using qWarning() and subsequent calls to \c isValid() will return false. diff --git a/src/testlib/qtest.h b/src/testlib/qtest.h index 27fe08e8f4..3b1ffb389e 100644 --- a/src/testlib/qtest.h +++ b/src/testlib/qtest.h @@ -95,7 +95,7 @@ template<> inline char *toString(const QByteArray &ba) template<> inline char *toString(const QBitArray &ba) { qsizetype size = ba.size(); - char *str = static_cast<char *>(malloc(size + 1)); + char *str = new char[size + 1]; for (qsizetype i = 0; i < size; ++i) str[i] = "01"[ba.testBit(i)]; str[size] = '\0'; diff --git a/src/testlib/qtestcase.qdoc b/src/testlib/qtestcase.qdoc index 9006d7b401..5088a812f3 100644 --- a/src/testlib/qtestcase.qdoc +++ b/src/testlib/qtestcase.qdoc @@ -43,55 +43,106 @@ true, execution continues. If not, a failure is recorded in the test log and the test won't be executed further. - \b {Note:} This macro can only be used in a test function that is invoked + You can use \l QVERIFY2() when it is practical and valuable to put additional + information into the test failure report. + + \note This macro can only be used in a test function that is invoked by the test framework. - Example: + For example, the following code shows this macro being used to verify that a + \l QSignalSpy object is valid: + \snippet code/src_qtestlib_qtestcase.cpp 0 - \sa QCOMPARE(), QTRY_VERIFY() + For more information about the failure, use \c QCOMPARE(x, y) instead of + \c QVERIFY(x == y), because it reports both the expected and actual value + when the comparison fails. + + \sa QCOMPARE(), QTRY_VERIFY(), QSignalSpy, QEXPECT_FAIL() */ /*! \macro QVERIFY2(condition, message) \relates QTest - The QVERIFY2() macro behaves exactly like QVERIFY(), except that it outputs - a verbose \a message when \a condition is false. The \a message is a plain - C string. + The QVERIFY2() macro behaves exactly like QVERIFY(), except that it reports + a \a message when \a condition is false. The \a message is a plain C string. + + The message can also be obtained from a function call that produces a plain + C string, such as qPrintable() applied to a QString, which may be built in + any of its usual ways, including applying \c {.args()} to format some data. Example: \snippet code/src_qtestlib_qtestcase.cpp 1 - \sa QVERIFY(), QCOMPARE() + For example, if you have a file object and you are testing its \c open() + function, you might write a test with a statement like: + + \snippet code/src_qtestlib_qtestcase.cpp 32 + + If this test fails, it will give no clue as to why the file failed to open: + + \c {FAIL! : tst_QFile::open_write() 'opened' returned FALSE. ()} + + If there is a more informative error message you could construct from the + values being tested, you can use \c QVERIFY2() to pass that message along + with your test condition, to provide a more informative message on failure: + + \snippet code/src_qtestlib_qtestcase.cpp 33 + + If this branch is being tested in the Qt CI system, the above detailed + failure message will be inserted into the summary posted to the code-review + system: + + \c {FAIL! : tst_QFile::open_write() 'opened' returned FALSE. + (open /tmp/qt.a3B42Cd: No space left on device)} + + \sa QVERIFY(), QCOMPARE(), QEXPECT_FAIL() */ /*! \macro QCOMPARE(actual, expected) \relates QTest - The QCOMPARE macro compares an \a actual value to an \a expected value using - the equals operator. If \a actual and \a expected are identical, execution + The QCOMPARE() macro compares an \a actual value to an \a expected value + using the equality operator. If \a actual and \a expected match, execution continues. If not, a failure is recorded in the test log and the test - won't be executed further. - - In the case of comparing floats and doubles, qFuzzyCompare() is used for - comparing. This means that comparing to 0 will likely fail. One solution - to this is to compare to 1, and add 1 to the produced output. - - QCOMPARE tries to output the contents of the values if the comparison fails, + function returns without attempting any later checks. + + Always respect QCOMPARE() parameter semantics. The first parameter passed to it + should always be the actual value produced by the code-under-test, while the + second parameter should always be the expected value. When the values don't + match, QCOMPARE() prints them with the labels \e Actual and \e Expected. + If the parameter order is swapped, debugging a failing test can be confusing. + + When comparing floating-point types (\c float, \c double, and \c qfloat16), + \l qFuzzyCompare() is used for finite values. Infinities match if they have + the same sign, and any NaN as actual value matches with any NaN as expected + value (even though NaN != NaN, even when they're identical). This means that + expecting 0 can fail when the actual value may be affected by rounding errors. + One solution to this is to offset both actual and expected values by adding + some suitable constant (such as 1). + + QCOMPARE() tries to output the contents of the values if the comparison fails, so it is visible from the test log why the comparison failed. - For your own classes, you can use \l QTest::toString() to format values for - outputting into the test log. + Example: + \snippet code/src_qtestlib_qtestcase.cpp 2 \note This macro can only be used in a test function that is invoked by the test framework. + For your own classes, you can use \l QTest::toString() to format values for + outputting into the test log. + Example: - \snippet code/src_qtestlib_qtestcase.cpp 2 + \snippet code/src_qtestlib_qtestcase.cpp 34 - \sa QVERIFY(), QTRY_COMPARE(), QTest::toString() + The return from \c toString() must be a \c {new char []}. That is, it shall + be released with \c delete[] (rather than \c free() or plain \c delete) once + the calling code is done with it. + + \sa QVERIFY(), QTRY_COMPARE(), QTest::toString(), QEXPECT_FAIL() */ /*! \macro QVERIFY_EXCEPTION_THROWN(expression, exceptiontype) @@ -127,7 +178,8 @@ \note This macro can only be used in a test function that is invoked by the test framework. - \sa QTRY_VERIFY(), QTRY_VERIFY2_WITH_TIMEOUT(), QVERIFY(), QCOMPARE(), QTRY_COMPARE() + \sa QTRY_VERIFY(), QTRY_VERIFY2_WITH_TIMEOUT(), QVERIFY(), QCOMPARE(), QTRY_COMPARE(), + QEXPECT_FAIL() */ @@ -141,7 +193,8 @@ \note This macro can only be used in a test function that is invoked by the test framework. - \sa QTRY_VERIFY_WITH_TIMEOUT(), QTRY_VERIFY2(), QVERIFY(), QCOMPARE(), QTRY_COMPARE() + \sa QTRY_VERIFY_WITH_TIMEOUT(), QTRY_VERIFY2(), QVERIFY(), QCOMPARE(), QTRY_COMPARE(), + QEXPECT_FAIL() */ /*! \macro QTRY_VERIFY2_WITH_TIMEOUT(condition, message, timeout) @@ -161,7 +214,8 @@ \note This macro can only be used in a test function that is invoked by the test framework. - \sa QTRY_VERIFY(), QTRY_VERIFY_WITH_TIMEOUT(), QVERIFY(), QCOMPARE(), QTRY_COMPARE() + \sa QTRY_VERIFY(), QTRY_VERIFY_WITH_TIMEOUT(), QVERIFY(), QCOMPARE(), QTRY_COMPARE(), + QEXPECT_FAIL() */ /*! \macro QTRY_VERIFY2(condition, message) @@ -181,7 +235,8 @@ \note This macro can only be used in a test function that is invoked by the test framework. - \sa QTRY_VERIFY2_WITH_TIMEOUT(), QTRY_VERIFY2(), QVERIFY(), QCOMPARE(), QTRY_COMPARE() + \sa QTRY_VERIFY2_WITH_TIMEOUT(), QTRY_VERIFY2(), QVERIFY(), QCOMPARE(), QTRY_COMPARE(), + QEXPECT_FAIL() */ /*! \macro QTRY_COMPARE_WITH_TIMEOUT(actual, expected, timeout) @@ -198,7 +253,7 @@ \note This macro can only be used in a test function that is invoked by the test framework. - \sa QTRY_COMPARE(), QCOMPARE(), QVERIFY(), QTRY_VERIFY() + \sa QTRY_COMPARE(), QCOMPARE(), QVERIFY(), QTRY_VERIFY(), QEXPECT_FAIL() */ /*! \macro QTRY_COMPARE(actual, expected) @@ -212,7 +267,8 @@ \note This macro can only be used in a test function that is invoked by the test framework. - \sa QTRY_COMPARE_WITH_TIMEOUT(), QCOMPARE(), QVERIFY(), QTRY_VERIFY() + \sa QTRY_COMPARE_WITH_TIMEOUT(), QCOMPARE(), QVERIFY(), QTRY_VERIFY(), + QEXPECT_FAIL() */ /*! \macro QFETCH(type, name) @@ -317,26 +373,55 @@ If called from a test function, the QSKIP() macro stops execution of the test without adding a failure to the test log. You can use it to skip tests that - wouldn't make sense in the current configuration. The text \a description is - appended to the test log and should contain an explanation of why the test - couldn't be executed. + wouldn't make sense in the current configuration. For example, a test of font + rendering may call QSKIP() if the needed fonts are not installed on the test + system. + + The text \a description is appended to the test log and should contain an + explanation of why the test couldn't be executed. + + If the test is data-driven, each call to QSKIP() in the test function will + skip only the current row of test data, so an unconditional call to QSKIP() + will produce one skip message in the test log for each row of test data. + + If called from an \c _data function, the QSKIP() macro will stop execution of + the \c _data function and will prevent execution of the associated test + function. This entirely omits a data-driven test. To omit individual rows, + make them conditional by using a simple \c{if (condition) newRow(...) << ...} + in the \c _data function, instead of using QSKIP() in the test function. + + If called from \c initTestCase_data(), the QSKIP() macro will skip all test + and \c _data functions. If called from \c initTestCase() when there is no + \c initTestCase_data(), or when it only sets up one row, QSKIP() will + likewise skip the whole test. However, if \c initTestCase_data() contains + more than one row, then \c initTestCase() is called (followed by each test + and finally the wrap-up) once per row of it. Therefore, a call to QSKIP() in + \c initTestCase() will merely skip all test functions for the current row of + global data, set up by \c initTestCase_data(). + + \note This macro can only be used in a test function or \c _data + function that is invoked by the test framework. - If the test is data-driven, each call to QSKIP() will skip only the current - row of test data, so an unconditional call to QSKIP will produce one skip - message in the test log for each row of test data. + Example: + \snippet code/src_qtestlib_qtestcase.cpp 8 - If called from an _data function, the QSKIP() macro will stop execution of - the _data function and will prevent execution of the associated test - function. + \section2 Skipping Known Bugs - If called from initTestCase() or initTestCase_data(), the QSKIP() macro will - skip all test and _data functions. + If a test exposes a known bug that will not be fixed immediately, use the + QEXPECT_FAIL() macro to document the failure and reference the bug tracking + identifier for the known issue. When the test is run, expected failures will + be marked as XFAIL in the test output and will not be counted as failures + when setting the test program's return code. If an expected failure does + not occur, the XPASS (unexpected pass) will be reported in the test output + and will be counted as a test failure. - \b {Note:} This macro can only be used in a test function or _data - function that is invoked by the test framework. + For known bugs, QEXPECT_FAIL() is better than QSKIP() because a developer + cannot fix the bug without an XPASS result reminding them that the test + needs to be updated too. If QSKIP() is used, there is no reminder to revise + or re-enable the test, without which subsequent regressions will not be + reported. - Example: - \snippet code/src_qtestlib_qtestcase.cpp 8 + \sa QEXPECT_FAIL(), {Select Appropriate Mechanisms to Exclude Tests} */ /*! \macro QEXPECT_FAIL(dataIndex, comment, mode) |