path: root/src/testlib
diff options
authorQt Forward Merge Bot <>2019-11-06 01:00:08 +0100
committerEdward Welbourne <>2019-11-06 11:41:19 +0100
commite3cc16e9fba045b3d195bc0e35dff4735bb6f59c (patch)
treea048d9ecb07ae175b4b357216c7e3e74d41c1271 /src/testlib
parent2e63d121839dae2794685001563783eea4c997f0 (diff)
parent7f0aaa88ef1db3840bd504d63a38d53032b8f802 (diff)
Merge remote-tracking branch 'origin/5.13' into 5.14
Conflicts: .qmake.conf mkspecs/features/mac/default_post.prf src/corelib/tools/qsimd_p.h src/platformsupport/fontdatabases/mac/ util/qfloat16-tables/gen_qfloat16_tables.cpp Change-Id: If48fa8a3bc3c983706b609a6d3822cb67c1352a4
Diffstat (limited to 'src/testlib')
6 files changed, 749 insertions, 50 deletions
diff --git a/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp b/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp
index 5f71828595..2dc4fe49ef 100644
--- a/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp
+++ b/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp
@@ -53,12 +53,12 @@ void wrapInFunction()
//! [0]
-QVERIFY(1 + 1 == 2);
+ QVERIFY(spy.isValid())
//! [0]
//! [1]
-QVERIFY2(1 + 1 == 2, "A breach in basic arithmetic occurred.");
+QVERIFY2(qIsNaN(0.0 / 0.0), "Ill-defined division produced unambiguous result.");
//! [1]
@@ -324,3 +324,28 @@ void TestQLocale::roundTripInt()
//! [31]
+//! [32]
+bool opened =;
+//! [32]
+//! [33]
+ qPrintable(QString("open %1: %2").arg(file.fileName()).arg(file.errorString()));
+//! [33]
+//! [34]
+namespace QTest {
+ template <> char *toString<MyType>(const MyType &t)
+ {
+ char *repr = new char[t.reprSize()];
+ t.writeRepr(repr);
+ return repr;
+ }
+//! [34]
diff --git a/src/testlib/doc/src/qt-webpages.qdoc b/src/testlib/doc/src/qt-webpages.qdoc
index 29a2589a4c..976435e668 100644
--- a/src/testlib/doc/src/qt-webpages.qdoc
+++ b/src/testlib/doc/src/qt-webpages.qdoc
@@ -1,6 +1,6 @@
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 The Qt Company Ltd.
** Contact:
** This file is part of the documentation of the Qt Toolkit.
@@ -25,10 +25,26 @@
- \externalpage
- \title Qt Labs
+ \externalpage
+ \title qtestlib-tools Announcement
- \externalpage
- \title qtestlib-tools Announcement
+ \externalpage
+ \title Froglogic Coco Code Coverage
+ \externalpage
+ \title gcov
+ \externalpage
+ \title Googletest Mocking (gMock) Framework
+ \externalpage
+ \title CMake/Testing With CTest
diff --git a/src/testlib/doc/src/qttest-best-practices.qdoc b/src/testlib/doc/src/qttest-best-practices.qdoc
new file mode 100644
index 0000000000..c7fee93c80
--- /dev/null
+++ b/src/testlib/doc/src/qttest-best-practices.qdoc
@@ -0,0 +1,533 @@
+** Copyright (C) 2019 The Qt Company Ltd.
+** Contact:
+** This file is part of the documentation of the Qt Toolkit.
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see For further
+** information use the contact form at
+** GNU Free Documentation License Usage
+** Alternatively, this file may be used under the terms of the GNU Free
+** Documentation License version 1.3 as published by the Free Software
+** Foundation and appearing in the file included in the packaging of
+** this file. Please review the following information to ensure
+** the GNU Free Documentation License version 1.3 requirements
+** will be met:
+ \page qttest-best-practices.qdoc
+ \title Qt Test Best Practices
+ \brief Guidelines for creating Qt tests.
+ We recommend that you add Qt tests for bug fixes and new features. Before
+ you try to fix a bug, add a \e {regression test} (ideally automatic) that
+ fails before the fix, exhibiting the bug, and passes after the fix. While
+ you're developing new features, add tests to verify that they work as
+ intended.
+ Conforming to a set of coding standards will make it more likely for
+ Qt autotests to work reliably in all environments. For example, some
+ tests need to read data from disk. If no standards are set for how this
+ is done, some tests won't be portable. For example, a test that assumes
+ its test-data files are in the current working directory only works for
+ an in-source build. In a shadow build (outside the source directory), the
+ test will fail to find its data.
+ The following sections contain guidelines for writing Qt tests:
+ \list
+ \li \l {General Principles}
+ \li \l {Writing Reliable Tests}
+ \li \l {Improving Test Output}
+ \li \l {Writing Testable Code}
+ \li \l {Setting up Test Machines}
+ \endlist
+ \section1 General Principles
+ The following sections provide general guidelines for writing unit tests:
+ \list
+ \li \l {Verify Tests}
+ \li \l {Give Test Functions Descriptive Names}
+ \li \l {Write Self-contained Test Functions}
+ \li \l {Test the Full Stack}
+ \li \l {Make Tests Complete Quickly}
+ \li \l {Use Data-driven Testing}
+ \li \l {Use Coverage Tools}
+ \li \l {Select Appropriate Mechanisms to Exclude Tests}
+ \li \l {Avoid Q_ASSERT}
+ \endlist
+ \section2 Verify Tests
+ Write and commit your tests along with your fix or new feature on a new
+ branch. Once you're done, you can check out the branch on which your work
+ is based, and then check out into this branch the test-files for your new
+ tests. This enables you to verify that the tests do fail on the prior
+ branch, and therefore actually do catch a bug or test a new feature.
+ For example, the workflow to fix a bug in the \c QDateTime class could be
+ like this if you use the Git version control system:
+ \list 1
+ \li Create a branch for your fix and test:
+ \c {git checkout -b fix-branch 5.14}
+ \li Write a test and fix the bug.
+ \li Build and test with both the fix and the new test, to verify that
+ the new test passes with the fix.
+ \li Add the fix and test to your branch:
+ \c {git add tests/auto/corelib/time/qdatetime/tst_qdatetime.cpp src/corelib/time/qdatetime.cpp}
+ \li Commit the fix and test to your branch:
+ \c {git commit -m 'Fix bug in QDateTime'}
+ \li To verify that the test actually catches something for which you
+ needed the fix, checkout the branch you based your own branch on:
+ \c {git checkout 5.14}
+ \li Checkout only the test file to the 5.14 branch:
+ \c {git checkout fix-branch -- tests/auto/corelib/time/qdatetime/tst_qdatetime.cpp}
+ Only the test is now on the fix-branch. The rest of the source tree
+ is still on 5.14.
+ \li Build and run the test to verify that it fails on 5.14, and
+ therefore does indeed catch a bug.
+ \li You can now return to the fix branch:
+ \c {git checkout fix-branch}
+ \li Alternatively, you can restore your work tree to a clean state on
+ 5.14:
+ \c{git checkout HEAD -- tests/auto/corelib/time/qdatetime/tst_qdatetime.cpp}
+ \endlist
+ When you're reviewing a change, you can adapt this workflow to check that
+ the change does indeed come with a test for a problem it does fix.
+ \section2 Give Test Functions Descriptive Names
+ Naming test cases is important. The test name appears in the failure report
+ for a test run. For data-driven tests, the name of the data row also appears
+ in the failure report. The names give those reading the report a first
+ indication of what has gone wrong.
+ Test function names should make it obvious what the function is trying to
+ test. Do not simply use the bug-tracking identifier, because the identifiers
+ become obsolete if the bug-tracker is replaced. Also, some bug-trackers may
+ not be accessible to all users. When the bug report may be of interest to
+ later readers of the test code, you can mention it in a comment alongside a
+ relevant part of the test.
+ Likewise, when writing data-driven tests, give descriptive names to the
+ test-cases, that indicate what aspect of the functionality each focuses on.
+ Do not simply number the test-case, or use bug-tracking identifiers. Someone
+ reading the test output will have no idea what the numbers or identifiers
+ mean. You can add a comment on the test-row that mentions the bug-tracking
+ identifier, when relevant.
+ \section2 Write Self-contained Test Functions
+ Within a test program, test functions should be independent of each other
+ and they should not rely upon previous test functions having been run. You
+ can check this by running the test function on its own with
+ \c {tst_foo testname}.
+ Do not re-use instances of the class under test in several tests. Test
+ instances (for example widgets) should not be member variables of the
+ tests, but preferably be instantiated on the stack to ensure proper
+ cleanup even if a test fails, so that tests do not interfere with
+ each other.
+ \section2 Test the Full Stack
+ If an API is implemented in terms of pluggable or platform-specific backends
+ that do the heavy-lifting, make sure to write tests that cover the
+ code-paths all the way down into the backends. Testing the upper layer API
+ parts using a mock backend is a nice way to isolate errors in the API layer
+ from the backends, but it is complementary to tests that run the actual
+ implementation with real-world data.
+ \section2 Make Tests Complete Quickly
+ Tests should not waste time by being unnecessarily repetitious, by using
+ inappropriately large volumes of test data, or by introducing needless
+ idle time.
+ This is particularly true for unit testing, where every second of extra
+ unit test execution time makes CI testing of a branch across multiple
+ targets take longer. Remember that unit testing is separate from load and
+ reliability testing, where larger volumes of test data and longer test
+ runs are expected.
+ Benchmark tests, which typically execute the same test multiple times,
+ should be located in a separate \c tests/benchmarks directory and they
+ should not be mixed with functional unit tests.
+ \section2 Use Data-driven Testing
+ Data-driven tests make it easier to add new tests for boundary conditions
+ found in later bug reports.
+ Using a data-driven test rather than testing several items in sequence in
+ a test saves repetition of very similar code and ensures later cases are
+ tested even when earlier ones fail. It also encourages systematic and
+ uniform testing, because the same tests are applied to each data sample.
+ \section2 Use Coverage Tools
+ Use a coverage tool such as \l {Froglogic Coco Code Coverage} or \l {gcov}
+ to help write tests that cover as many statements, branches, and conditions
+ as possible in the function or class being tested. The earlier this is done
+ in the development cycle for a new feature, the easier it will be to catch
+ regressions later when the code is refactored.
+ \section2 Select Appropriate Mechanisms to Exclude Tests
+ It is important to select the appropriate mechanism to exclude inapplicable
+ tests: \l QSKIP(), using conditional statements to exclude parts of a test
+ function, or not building the test for a particular platform.
+ Use QSKIP() to handle cases where a whole test function is found at run-time
+ to be inapplicable in the current test environment. When just a part of a
+ test function is to be skipped, a conditional statement can be used,
+ optionally with a \c qDebug() call to report the reason for skipping the
+ inapplicable part.
+ Test functions or data rows of a data-driven test can be limited to
+ particular platforms, or to particular features being enabled using
+ \c{#if}. However, beware of \l moc limitations when using \c{#if} to
+ skip test functions. The \c moc preprocessor does not have access to
+ all the \c builtin macros of the compiler that are often used for
+ feature detection of the compiler. Therefore, \c moc might get a different
+ result for a preprocessor condition from that seen by the rest of your
+ code. This may result in \c moc generating meta-data for a test slot that
+ the actual compiler skips, or omitting the meta-data for a test slot that
+ is actually compiled into the class. In the first case, the test will
+ attempt to run a slot that is not implemented. In the second case, the
+ test will not attempt to run a test slot even though it should.
+ If an entire test program is inapplicable for a specific platform or
+ unless a particular feature is enabled, the best approach is to use the
+ parent directory's \c .pro file to avoid building the test. For example,
+ if the \c tests/auto/gui/someclass test is not valid for \macOS, add the
+ following line to \c tests/auto/
+ \badcode
+ mac*: SUBDIRS -= someclass
+ \endcode
+ \section2 Avoid Q_ASSERT
+ The \l Q_ASSERT macro causes a program to abort whenever the asserted
+ condition is \c false, but only if the software was built in debug mode.
+ In both release and debug-and-release builds, \c Q_ASSERT does nothing.
+ \c Q_ASSERT should be avoided because it makes tests behave differently
+ depending on whether a debug build is being tested, and because it causes
+ a test to abort immediately, skipping all remaining test functions and
+ returning incomplete or malformed test results.
+ It also skips any tear-down or tidy-up that was supposed to happen at the
+ end of the test, and might therefore leave the workspace in an untidy state,
+ which might cause complications for further tests.
+ Instead of \c Q_ASSERT, the \l QCOMPARE() or \l QVERIFY() macro variants
+ should be used. They cause the current test to report a failure and
+ terminate, but allow the remaining test functions to be executed and the
+ entire test program to terminate normally. \l Q_VERIFY2() even allows a
+ descriptive error message to be recorded in the test log.
+ \section1 Writing Reliable Tests
+ The following sections provide guidelines for writing reliable tests:
+ \list
+ \li \l {Avoid Side-effects in Verification Steps}
+ \li \l {Avoid Fixed Timeouts}
+ \li \l {Beware of Timing-dependent Behavior}
+ \li \l {Avoid Bitmap Capture and Comparison}
+ \endlist
+ \section2 Avoid Side-effects in Verification Steps
+ When performing verification steps in an autotest using \l QCOMPARE(),
+ \l QVERIFY(), and so on, side-effects should be avoided. Side-effects
+ in verification steps can make a test difficult to understand. Also,
+ they can easily break a test in ways that are difficult to diagnose
+ when the test is changed to use \l QTRY_VERIFY(), \l QTRY_COMPARE() or
+ \l QBENCHMARK(). These can execute the passed expression multiple times,
+ thus repeating any side-effects.
+ When side-effects are unavoidable, ensure that the prior state is restored
+ at the end of the test function, even if the test fails. This commonly
+ requires use of an RAII (resource acquisition is initialization) class
+ that restores state when the function returns, or a \l cleanup() method.
+ Do not simply put the restoration code at the end of the test. If part of
+ the test fails, such code will be skipped and the prior state will not be
+ restored.
+ \section2 Avoid Fixed Timeouts
+ Avoid using hard-coded timeouts, such as QTest::qWait() to wait for some
+ conditions to become true. Consider using the \l QtSignalSpy class,
+ the \l QTRY_VERIFY() or \l QTRY_COMPARE() macros, or the \c QtSignalSpy
+ class in conjunction with the \c QTRY_ macro variants.
+ The \c qWait() function can be used to set a delay for a fixed period
+ between performing some action and waiting for some asynchronous behavior
+ triggered by that action to be completed. For example, changing the state
+ of a widget and then waiting for the widget to be repainted. However,
+ such timeouts often cause failures when a test written on a workstation is
+ executed on a device, where the expected behavior might take longer to
+ complete. Increasing the fixed timeout to a value several times larger
+ than needed on the slowest test platform is not a good solution, because
+ it slows down the test run on all platforms, particularly for table-driven
+ tests.
+ If the code under test issues Qt signals on completion of the asynchronous
+ behavior, a better approach is to use the \l QSignalSpy class to notify
+ the test function that the verification step can now be performed.
+ If there are no Qt signals, use the \c QTRY_COMPARE() and \c QTRY_VERIFY()
+ macros, which periodically test a specified condition until it becomes true
+ or some maximum timeout is reached. These macros prevent the test from
+ taking longer than necessary, while avoiding breakages when tests are
+ written on workstations and later executed on embedded platforms.
+ If there are no Qt signals, and you are writing the test as part of
+ developing a new API, consider whether the API could benefit from the
+ addition of a signal that reports the completion of the asynchronous
+ behavior.
+ \section2 Beware of Timing-dependent Behavior
+ Some test strategies are vulnerable to timing-dependent behavior of certain
+ classes, which can lead to tests that fail only on certain platforms or that
+ do not return consistent results.
+ One example of this is text-entry widgets, which often have a blinking
+ cursor that can make comparisons of captured bitmaps succeed or fail
+ depending on the state of the cursor when the bitmap is captured. This,
+ in turn, may depend on the speed of the machine executing the test.
+ When testing classes that change their state based on timer events, the
+ timer-based behavior needs to be taken into account when performing
+ verification steps. Due to the variety of timing-dependent behavior, there
+ is no single generic solution to this testing problem.
+ For text-entry widgets, potential solutions include disabling the cursor
+ blinking behavior (if the API provides that feature), waiting for the
+ cursor to be in a known state before capturing a bitmap (for example, by
+ subscribing to an appropriate signal if the API provides one), or
+ excluding the area containing the cursor from the bitmap comparison.
+ \section2 Avoid Bitmap Capture and Comparison
+ While verifying test results by capturing and comparing bitmaps is sometimes
+ necessary, it can be quite fragile and labor-intensive.
+ For example, a particular widget may have different appearance on different
+ platforms or with different widget styles, so reference bitmaps may need to
+ be created multiple times and then maintained in the future as Qt's set of
+ supported platforms evolves. Making changes that affect the bitmap thus
+ means having to recreate the expected bitmaps on each supported platform,
+ which would require access to each platform.
+ Bitmap comparisons can also be influenced by factors such as the test
+ machine's screen resolution, bit depth, active theme, color scheme,
+ widget style, active locale (currency symbols, text direction, and so
+ on), font size, transparency effects, and choice of window manager.
+ Where possible, use programmatic means, such as verifying properties of
+ objects and variables, instead of capturing and comparing bitmaps.
+ \section1 Improving Test Output
+ The following sections provide guidelines for producing readable and
+ helpful test output:
+ \list
+ \li \l {Explicitly Ignore Expected Warnings}
+ \li \l {Avoid Printing Debug Messages from Autotests}
+ \li \l {Write Well-structured Diagnostic Code}
+ \endlist
+ \section2 Explicitly Ignore Expected Warnings
+ If a test is expected to cause Qt to output a warning or debug message
+ on the console, it should call \l QTest::ignoreMessage() to filter that
+ message out of the test output and to fail the test if the message is
+ not output.
+ If such a message is only output when Qt is built in debug mode, use
+ \l QLibraryInfo::isDebugBuild() to determine whether the Qt libraries
+ were built in debug mode. Using \c{#ifdef QT_DEBUG} is not enough, as
+ it will only tell you whether the test was built in debug mode, and
+ that does not guarantee that the Qt libraries were also built in debug
+ mode.
+ \section2 Avoid Printing Debug Messages from Autotests
+ Autotests should not produce any unhandled warning or debug messages.
+ This will allow the CI Gate to treat new warning or debug messages as
+ test failures.
+ Adding debug messages during development is fine, but these should be
+ either disabled or removed before a test is checked in.
+ \section2 Write Well-structured Diagnostic Code
+ Any diagnostic output that would be useful if a test fails should be part
+ of the regular test output rather than being commented-out, disabled by
+ preprocessor directives, or enabled only in debug builds. If a test fails
+ during continuous integration, having all of the relevant diagnostic output
+ in the CI logs could save you a lot of time compared to enabling the
+ diagnostic code and testing again. Epecially, if the failure was on a
+ platform that you don't have on your desktop.
+ Diagnostic messages in tests should use Qt's output mechanisms, such as
+ \c qDebug() and \c qWarning(), rather than \c stdio.h or \c iostream.h output
+ mechanisms. The latter bypass Qt's message handling and prevent the
+ \c -silent command-line option from suppressing the diagnostic messages.
+ This could result in important failure messages being hidden in a large
+ volume of debugging output.
+ \section1 Writing Testable Code
+ The following sections provide guidelines for writing code that is easy to
+ test:
+ \list
+ \li \l {Break Dependencies}
+ \li \l {Compile All Classes into Libraries}
+ \endlist
+ \section2 Break Dependencies
+ The idea of unit testing is to use every class in isolation. Since many
+ classes instantiate other classes, it is not possible to instantiate one
+ class separately. Therefore, you should use a technique called
+ \e {dependency injection} that separates object creation from object use.
+ A factory is responsible for building object trees. Other objects manipulate
+ these objects through abstract interfaces.
+ This technique works well for data-driven applications. For GUI
+ applications, this approach can be difficult as objects are frequently
+ created and destructed. To verify the correct behavior of classes that
+ depend on abstract interfaces, \e mocking can be used. For example, see
+ \l {Googletest Mocking (gMock) Framework}.
+ \section2 Compile All Classes into Libraries
+ In small to medium sized projects, a build script typically lists all
+ source files and then compiles the executable in one go. This means that
+ the build scripts for the tests must list the needed source files again.
+ It is easier to list the source files and the headers only once in a
+ script to build a static library. Then the \c main() function will be
+ linked against the static library to build the executable and the tests
+ will be linked against the static libraries.
+ For projects where the same source files are used in building several
+ programs, it may be more appropriate to build the shared classes into
+ a dynamically-linked (or shared object) library that each program,
+ including the test programs, can load at run-time. Again, having the
+ compiled code in a library helps to avoid duplication in the description
+ of which components to combine to make the various programs.
+ \section1 Setting up Test Machines
+ The following sections discuss common problems caused by test machine setup:
+ \list
+ \li \l {Screen Savers}
+ \li \l {System Dialogs}
+ \li \l {Display Usage}
+ \li \l {Window Managers}
+ \endlist
+ All of these problems can typically be solved by the judicious use of
+ virtualisation.
+ \section2 Screen Savers
+ Screen savers can interfere with some of the tests for GUI classes, causing
+ unreliable test results. Screen savers should be disabled to ensure that
+ test results are consistent and reliable.
+ \section2 System Dialogs
+ Dialogs displayed unexpectedly by the operating system or other running
+ applications can steal input focus from widgets involved in an autotest,
+ causing unreproducible failures.
+ Examples of typical problems include online update notification dialogs
+ on macOS, false alarms from virus scanners, scheduled tasks such as virus
+ signature updates, software updates pushed out to workstations, and chat
+ programs popping up windows on top of the stack.
+ \section2 Display Usage
+ Some tests use the test machine's display, mouse, and keyboard, and can
+ thus fail if the machine is being used for something else at the same
+ time or if multiple tests are run in parallel.
+ The CI system uses dedicated test machines to avoid this problem, but if
+ you don't have a dedicated test machine, you may be able to solve this
+ problem by running the tests on a second display.
+ On Unix, one can also run the tests on a nested or virtual X-server, such as
+ Xephyr. For example, to run the entire set of tests on Xephyr, execute the
+ following commands:
+ \code
+ Xephyr :1 -ac -screen 1920x1200 >/dev/null 2>&1 &
+ sleep 5
+ DISPLAY=:1 icewm >/dev/null 2>&1 &
+ cd tests/auto
+ make
+ DISPLAY=:1 make -k -j1 check
+ \endcode
+ Users of NVIDIA binary drivers should note that Xephyr might not be able to
+ provide GLX extensions. Forcing Mesa libGL might help:
+ \code
+ export LD_PRELOAD=/usr/lib/mesa-diverted/x86_64-linux-gnu/
+ \endcode
+ However, when tests are run on Xephyr and the real X-server with different
+ libGL versions, the QML disk cache can make the tests crash. To avoid this,
+ Alternatively, use the offscreen plugin:
+ \code
+ TESTARGS="-platform offscreen" make check -k -j1
+ \endcode
+ \section2 Window Managers
+ On Unix, at least two autotests (\c tst_examples and \c tst_gestures)
+ require a window manager to be running. Therefore, if running these
+ tests under a nested X-server, you must also run a window manager
+ in that X-server.
+ Your window manager must be configured to position all windows on the
+ display automatically. Some windows managers, such as Tab Window Manager
+ (twm), have a mode for manually positioning new windows, and this prevents
+ the test suite from running without user interaction.
+ \note Tab Window Manager is not suitable for running the full suite of
+ Qt autotests, as the \c tst_gestures autotest causes it to forget its
+ configuration and revert to manual window placement.
diff --git a/src/testlib/doc/src/qttest-index.qdoc b/src/testlib/doc/src/qttest-index.qdoc
index 23be46b431..e31f232069 100644
--- a/src/testlib/doc/src/qttest-index.qdoc
+++ b/src/testlib/doc/src/qttest-index.qdoc
@@ -1,6 +1,6 @@
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 The Qt Company Ltd.
** Contact:
** This file is part of the documentation of the Qt Toolkit.
@@ -56,6 +56,7 @@
\li \l{Qt Test Overview}
+ \li \l{Qt Test Best Practices}
\li \l{Qt Test Tutorial}
diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc
index bb379fe029..19d871d404 100644
--- a/src/testlib/doc/src/qttestlib-manual.qdoc
+++ b/src/testlib/doc/src/qttestlib-manual.qdoc
@@ -102,6 +102,20 @@
\li \c{cleanup()} will be called after every test function.
+ Use \c initTestCase() for preparing the test. Every test should leave the
+ system in a usable state, so it can be run repeatedly. Cleanup operations
+ should be handled in \c cleanupTestCase(), so they get run even if the test
+ fails.
+ Use \c init() for preparing a test function. Every test function should
+ leave the system in a usable state, so it can be run repeatedly. Cleanup
+ operations should be handled in \c cleanup(), so they get run even if the
+ test function fails and exits early.
+ Alternatively, you can use RAII (resource acquisition is initialization),
+ with cleanup operations called in destructors, to ensure they happen when
+ the test function returns and the object moves out of scope.
If \c{initTestCase()} fails, no test function will be executed. If \c{init()} fails,
the following test function will not be executed, the test will proceed to the next
test function.
@@ -308,10 +322,35 @@
\section1 Creating a Benchmark
To create a benchmark, follow the instructions for creating a test and then add a
- QBENCHMARK macro to the test function that you want to benchmark.
+ \l QBENCHMARK macro or \l QTest::setBenchmarkResult() to the test function that
+ you want to benchmark. In the following code snippet, the macro is used:
\snippet code/doc_src_qtestlib.cpp 12
+ A test function that measures performance should contain either a single
+ \c QBENCHMARK macro or a single call to \c setBenchmarkResult(). Multiple
+ occurrences make no sense, because only one performance result can be
+ reported per test function, or per data tag in a data-driven setup.
+ Avoid changing the test code that forms (or influences) the body of a
+ \c QBENCHMARK macro, or the test code that computes the value passed to
+ \c setBenchmarkResult(). Differences in successive performance results
+ should ideally be caused only by changes to the product you are testing.
+ Changes to the test code can potentially result in misleading report of
+ a change in performance. If you do need to change the test code, make
+ that clear in the commit message.
+ In a performance test function, the \c QBENCHMARK or \c setBenchmarkResult()
+ should be followed by a verification step using \l QCOMPARE(), \l QVERIFY(),
+ and so on. You can then flag a performance result as \e invalid if another
+ code path than the intended one was measured. A performance analysis tool
+ can use this information to filter out invalid results.
+ For example, an unexpected error condition will typically cause the program
+ to bail out prematurely from the normal program execution, and thus falsely
+ show a dramatic performance increase.
+ \section2 Selecting the Measurement Back-end
The code inside the QBENCHMARK macro will be measured, and possibly also repeated
several times in order to get an accurate measurement. This depends on the selected
measurement back-end. Several back-ends are available. They can be selected on the
diff --git a/src/testlib/qtestcase.qdoc b/src/testlib/qtestcase.qdoc
index 9006d7b401..5088a812f3 100644
--- a/src/testlib/qtestcase.qdoc
+++ b/src/testlib/qtestcase.qdoc
@@ -43,55 +43,106 @@
true, execution continues. If not, a failure is recorded in the test log
and the test won't be executed further.
- \b {Note:} This macro can only be used in a test function that is invoked
+ You can use \l QVERIFY2() when it is practical and valuable to put additional
+ information into the test failure report.
+ \note This macro can only be used in a test function that is invoked
by the test framework.
- Example:
+ For example, the following code shows this macro being used to verify that a
+ \l QSignalSpy object is valid:
\snippet code/src_qtestlib_qtestcase.cpp 0
+ For more information about the failure, use \c QCOMPARE(x, y) instead of
+ \c QVERIFY(x == y), because it reports both the expected and actual value
+ when the comparison fails.
/*! \macro QVERIFY2(condition, message)
\relates QTest
- The QVERIFY2() macro behaves exactly like QVERIFY(), except that it outputs
- a verbose \a message when \a condition is false. The \a message is a plain
- C string.
+ The QVERIFY2() macro behaves exactly like QVERIFY(), except that it reports
+ a \a message when \a condition is false. The \a message is a plain C string.
+ The message can also be obtained from a function call that produces a plain
+ C string, such as qPrintable() applied to a QString, which may be built in
+ any of its usual ways, including applying \c {.args()} to format some data.
\snippet code/src_qtestlib_qtestcase.cpp 1
+ For example, if you have a file object and you are testing its \c open()
+ function, you might write a test with a statement like:
+ \snippet code/src_qtestlib_qtestcase.cpp 32
+ If this test fails, it will give no clue as to why the file failed to open:
+ \c {FAIL! : tst_QFile::open_write() 'opened' returned FALSE. ()}
+ If there is a more informative error message you could construct from the
+ values being tested, you can use \c QVERIFY2() to pass that message along
+ with your test condition, to provide a more informative message on failure:
+ \snippet code/src_qtestlib_qtestcase.cpp 33
+ If this branch is being tested in the Qt CI system, the above detailed
+ failure message will be inserted into the summary posted to the code-review
+ system:
+ \c {FAIL! : tst_QFile::open_write() 'opened' returned FALSE.
+ (open /tmp/qt.a3B42Cd: No space left on device)}
/*! \macro QCOMPARE(actual, expected)
\relates QTest
- The QCOMPARE macro compares an \a actual value to an \a expected value using
- the equals operator. If \a actual and \a expected are identical, execution
+ The QCOMPARE() macro compares an \a actual value to an \a expected value
+ using the equality operator. If \a actual and \a expected match, execution
continues. If not, a failure is recorded in the test log and the test
- won't be executed further.
- In the case of comparing floats and doubles, qFuzzyCompare() is used for
- comparing. This means that comparing to 0 will likely fail. One solution
- to this is to compare to 1, and add 1 to the produced output.
- QCOMPARE tries to output the contents of the values if the comparison fails,
+ function returns without attempting any later checks.
+ Always respect QCOMPARE() parameter semantics. The first parameter passed to it
+ should always be the actual value produced by the code-under-test, while the
+ second parameter should always be the expected value. When the values don't
+ match, QCOMPARE() prints them with the labels \e Actual and \e Expected.
+ If the parameter order is swapped, debugging a failing test can be confusing.
+ When comparing floating-point types (\c float, \c double, and \c qfloat16),
+ \l qFuzzyCompare() is used for finite values. Infinities match if they have
+ the same sign, and any NaN as actual value matches with any NaN as expected
+ value (even though NaN != NaN, even when they're identical). This means that
+ expecting 0 can fail when the actual value may be affected by rounding errors.
+ One solution to this is to offset both actual and expected values by adding
+ some suitable constant (such as 1).
+ QCOMPARE() tries to output the contents of the values if the comparison fails,
so it is visible from the test log why the comparison failed.
- For your own classes, you can use \l QTest::toString() to format values for
- outputting into the test log.
+ Example:
+ \snippet code/src_qtestlib_qtestcase.cpp 2
\note This macro can only be used in a test function that is invoked
by the test framework.
+ For your own classes, you can use \l QTest::toString() to format values for
+ outputting into the test log.
- \snippet code/src_qtestlib_qtestcase.cpp 2
+ \snippet code/src_qtestlib_qtestcase.cpp 34
- \sa QVERIFY(), QTRY_COMPARE(), QTest::toString()
+ The return from \c toString() must be a \c {new char []}. That is, it shall
+ be released with \c delete[] (rather than \c free() or plain \c delete) once
+ the calling code is done with it.
+ \sa QVERIFY(), QTRY_COMPARE(), QTest::toString(), QEXPECT_FAIL()
/*! \macro QVERIFY_EXCEPTION_THROWN(expression, exceptiontype)
@@ -127,7 +178,8 @@
\note This macro can only be used in a test function that is invoked
by the test framework.
@@ -141,7 +193,8 @@
\note This macro can only be used in a test function that is invoked
by the test framework.
/*! \macro QTRY_VERIFY2_WITH_TIMEOUT(condition, message, timeout)
@@ -161,7 +214,8 @@
\note This macro can only be used in a test function that is invoked
by the test framework.
/*! \macro QTRY_VERIFY2(condition, message)
@@ -181,7 +235,8 @@
\note This macro can only be used in a test function that is invoked
by the test framework.
/*! \macro QTRY_COMPARE_WITH_TIMEOUT(actual, expected, timeout)
@@ -198,7 +253,7 @@
\note This macro can only be used in a test function that is invoked
by the test framework.
/*! \macro QTRY_COMPARE(actual, expected)
@@ -212,7 +267,8 @@
\note This macro can only be used in a test function that is invoked
by the test framework.
/*! \macro QFETCH(type, name)
@@ -317,26 +373,55 @@
If called from a test function, the QSKIP() macro stops execution of the test
without adding a failure to the test log. You can use it to skip tests that
- wouldn't make sense in the current configuration. The text \a description is
- appended to the test log and should contain an explanation of why the test
- couldn't be executed.
+ wouldn't make sense in the current configuration. For example, a test of font
+ rendering may call QSKIP() if the needed fonts are not installed on the test
+ system.
+ The text \a description is appended to the test log and should contain an
+ explanation of why the test couldn't be executed.
+ If the test is data-driven, each call to QSKIP() in the test function will
+ skip only the current row of test data, so an unconditional call to QSKIP()
+ will produce one skip message in the test log for each row of test data.
+ If called from an \c _data function, the QSKIP() macro will stop execution of
+ the \c _data function and will prevent execution of the associated test
+ function. This entirely omits a data-driven test. To omit individual rows,
+ make them conditional by using a simple \c{if (condition) newRow(...) << ...}
+ in the \c _data function, instead of using QSKIP() in the test function.
+ If called from \c initTestCase_data(), the QSKIP() macro will skip all test
+ and \c _data functions. If called from \c initTestCase() when there is no
+ \c initTestCase_data(), or when it only sets up one row, QSKIP() will
+ likewise skip the whole test. However, if \c initTestCase_data() contains
+ more than one row, then \c initTestCase() is called (followed by each test
+ and finally the wrap-up) once per row of it. Therefore, a call to QSKIP() in
+ \c initTestCase() will merely skip all test functions for the current row of
+ global data, set up by \c initTestCase_data().
+ \note This macro can only be used in a test function or \c _data
+ function that is invoked by the test framework.
- If the test is data-driven, each call to QSKIP() will skip only the current
- row of test data, so an unconditional call to QSKIP will produce one skip
- message in the test log for each row of test data.
+ Example:
+ \snippet code/src_qtestlib_qtestcase.cpp 8
- If called from an _data function, the QSKIP() macro will stop execution of
- the _data function and will prevent execution of the associated test
- function.
+ \section2 Skipping Known Bugs
- If called from initTestCase() or initTestCase_data(), the QSKIP() macro will
- skip all test and _data functions.
+ If a test exposes a known bug that will not be fixed immediately, use the
+ QEXPECT_FAIL() macro to document the failure and reference the bug tracking
+ identifier for the known issue. When the test is run, expected failures will
+ be marked as XFAIL in the test output and will not be counted as failures
+ when setting the test program's return code. If an expected failure does
+ not occur, the XPASS (unexpected pass) will be reported in the test output
+ and will be counted as a test failure.
- \b {Note:} This macro can only be used in a test function or _data
- function that is invoked by the test framework.
+ For known bugs, QEXPECT_FAIL() is better than QSKIP() because a developer
+ cannot fix the bug without an XPASS result reminding them that the test
+ needs to be updated too. If QSKIP() is used, there is no reminder to revise
+ or re-enable the test, without which subsequent regressions will not be
+ reported.
- Example:
- \snippet code/src_qtestlib_qtestcase.cpp 8
+ \sa QEXPECT_FAIL(), {Select Appropriate Mechanisms to Exclude Tests}
/*! \macro QEXPECT_FAIL(dataIndex, comment, mode)