summaryrefslogtreecommitdiffstats
path: root/src/testlib/qabstracttestlogger.cpp
blob: cb9a0493570ed43deb67c1d13b2c75e7767cb009 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
/****************************************************************************
**
** Copyright (C) 2021 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtTest module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include <QtTest/private/qabstracttestlogger_p.h>
#include <QtTest/qtestassert.h>
#include <qtestresult_p.h>

#include <QtCore/qbytearray.h>
#include <QtCore/qstring.h>

#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>

#ifndef Q_OS_WIN
#include <unistd.h>
#endif

#ifdef Q_OS_ANDROID
#include <sys/stat.h>
#endif

QT_BEGIN_NAMESPACE
/*!
    \internal
    \class QAbstractTestLogger
    \inmodule QtTest
    \brief Base class for test loggers

    Implementations of logging for QtTest should implement all pure virtual
    methods of this class and may implement the other virtual methods. This
    class's documentation of each virtual method sets out how those
    implementations are invoked by the QtTest code and offers guidance on how
    the logging class should use the data. Actual implementations may have
    different requirements - such as a file format with a defined schema, or a
    target audience to serve - that affect how it interprets that guidance.
*/

/*!
    \enum QAbstractTestLogger::IncidentTypes

    \value Pass The test ran to completion successfully.
    \value XFail The test failed a check that is known to fail; this failure
           does not prevent successful completion of the test and may be
           followed by further checks.
    \value Fail The test fails.
    \value XPass A check which was expected to fail actually passed. This is
           counted as a failure, as it means whatever caused the known failure
           no longer does, so the test needs an update.
    \value Skip The current test ended prematurely, skipping some checks.
    \value BlacklistedPass As Pass but the test was blacklisted.
    \value BlacklistedXFail As XFail but the test was blacklisted.
    \value BlacklistedFail As Fail but the test was blacklisted.
    \value BlacklistedXPass As XPass but the test was blacklisted.

    A test may also skip (see \l {QAbstractTestLogger::}{MessageTypes}). The
    first of skip, Fail, XPass or the blacklisted equivalents of the last two to
    arise is decisive for the outcome of the test: loggers which should only
    report one outcome should thus record that as the outcome and either ignore
    later incidents (or skips) in the same run of the test function or map them
    to some form of message.

    \note tests can be "blacklisted" when they are known to fail
    unreliably. When testing is used to decide whether a change to the code
    under test is acceptable, such failures are not automatic grounds for
    rejecting the change, if the unreliable failure was known before the
    change. QTest::qExec(), as a result, only returns a failing status code if
    some non-blacklisted test failed. Logging backends may reasonably report a
    blacklisted result just as they would report the non-blacklisted equivalent,
    optionally with some annotation to indicate that the result should not be
    taken as damning evidence against recent changes to the code under test.

    \sa QAbstractTestLogger::addIncident()
*/

/*!
    \enum QAbstractTestLogger::MessageTypes

    The members whose names begin with \c Q describe messages that originate in
    calls, by the test or code under test, to Qt logging functions (implemented
    as macros) whose names are similar, with a \c q in place of the leading \c
    Q. The other members describe messages generated internally by QtTest.

    \value QInfo An informational message from qInfo().
    \value QWarning A warning from qWarning().
    \value QDebug A debug message from qDebug().
    \value QCritical A critical error from qCritical().
    \value QFatal A fatal error from qFatal(), or an unrecognised message from
           the Qt logging functions.
    \value Info Messages QtTest generates as requested by the \c{-v1} or \c{-v2}
           command-line option being specified when running the test.
    \value Warn A warning generated internally by QtTest

    \note For these purposes, some utilities provided by QtTestlib as helper
    functions to facilitate testing - such as \l QSignalSpy, \l
    QTestAccessibility, \l QTest::qExtractTestData(), and the facilities to
    deliver artificial mouse and keyboard events - are treated as test code,
    rather than internal to QtTest; they call \l qWarning() and friends rather
    than using the internal logging infrastructure, so that \l
    QTest::ignoreMessage() can be used to anticipate the messages.

    \sa QAbstractTestLogger::addMessage()
*/

/*!
    Constructs the base-class parts of the logger.

    Derived classes should pass this base-constructor the \a filename of the
    file to which they shall log test results, or \nullptr to write to standard
    output. The protected member \c stream is set to the open file descriptor.
*/
QAbstractTestLogger::QAbstractTestLogger(const char *filename)
{
    if (!filename) {
        stream = stdout;
        return;
    }
#if defined(_MSC_VER)
    if (::fopen_s(&stream, filename, "wt")) {
#else
    stream = ::fopen(filename, "wt");
    if (!stream) {
#endif
        fprintf(stderr, "Unable to open file for logging: %s\n", filename);
        ::exit(1);
    }
#ifdef Q_OS_ANDROID
    else {
        // Make sure output is world-readable on Android
        ::chmod(filename, 0666);
    }
#endif
}

/*!
    Destroys the logger object.

    If the protected \c stream is not standard output, it is closed. In any
    case it is cleared.
*/
QAbstractTestLogger::~QAbstractTestLogger()
{
    QTEST_ASSERT(stream);
    if (stream != stdout)
        fclose(stream);
    stream = nullptr;
}

/*!
    Returns true if the \c output stream is standard output.
*/
bool QAbstractTestLogger::isLoggingToStdout() const
{
    return stream == stdout;
}

/*!
    Helper utility to blot out unprintable characters in \a str.

    Takes a \c{'\0'}-terminated mutable string and changes any characters of it
    that are not suitable for printing to \c{'?'} characters.
*/
void QAbstractTestLogger::filterUnprintable(char *str) const
{
    unsigned char *idx = reinterpret_cast<unsigned char *>(str);
    while (*idx) {
        if (((*idx < 0x20 && *idx != '\n' && *idx != '\t') || *idx == 0x7f))
            *idx = '?';
        ++idx;
    }
}

/*!
    Convenience method to write \a msg to the output stream.

    The output \a msg must be a \c{'\0'}-terminated string (and not \nullptr).
    A copy of it is passed to \l filterUnprintable() and the result written to
    the output \c stream, which is then flushed.
*/
void QAbstractTestLogger::outputString(const char *msg)
{
    QTEST_ASSERT(stream);
    QTEST_ASSERT(msg);

    char *filtered = new char[strlen(msg) + 1];
    strcpy(filtered, msg);
    filterUnprintable(filtered);

    ::fputs(filtered, stream);
    ::fflush(stream);

    delete [] filtered;
}

/*!
    Called before the start of a test run.

    This virtual method is called before the first tests are run. A logging
    implementation might open a file, write some preamble, or prepare in other
    ways, such as setting up initial values of variables. It can use the usual
    Qt logging infrastucture, since it is also called before QtTest installs its
    own custom message handler.

    \sa stopLogging()
*/
void QAbstractTestLogger::startLogging()
{
}

/*!
    Called after the end of a test run.

    This virtual method is called after all tests have run. A logging
    implementation might collate information gathered from the run, write a
    summary, or close a file. It can use the usual Qt logging infrastucture,
    since it is also called after QtTest has restored the default message
    handler it replaced with its own custom message handler.

    \sa startLogging()
*/
void QAbstractTestLogger::stopLogging()
{
}

/*!
    \fn void QAbstractTestLogger::enterTestFunction(const char *function)

    This virtual method is called before each test function is invoked. It is
    passed the name of the test function (without its class prefix) as \a
    function. It is likewise called for \c{initTestCase()} at the start of
    testing, after \l startLogging(), and for \c{cleanupTestCase()} at the end
    of testing, in each case passing the name of the function. It is also called
    with \nullptr as \a function after the last of these functions, or in the
    event of an early end to testing, before \l stopLogging().

    For data-driven test functions, this is called only once, before the data
    function is called to set up the table of datasets and the test is run with
    its first dataset.

    Every logging implementation must implement this method. It shall typically
    need to record the name of the function for later use in log messages.

    \sa leaveTestFunction(), enterTestData()
*/
/*!
    \fn void QAbstractTestLogger::leaveTestFunction()

    This virtual method is called after a test function has completed, to match
    \l enterTestFunction(). For data-driven test functions, this is called only
    once, after the test is run with its last dataset.

    Every logging implementation must implement this method. In some cases it
    may be called more than once without an intervening call to \l
    enterTestFunction(). In such cases, the implementation should ignore these
    later calls, until the next call to enterTestFunction().

    \sa enterTestFunction(), enterTestData()
*/
/*!
    \fn void QAbstractTestLogger::enterTestData(QTestData *)

    This virtual method is called before and after each call to a test
    function. For a data-driven test, the call before is passed the name of the
    test data row. This may combine a global data row name with a local data row
    name. For non-data-driven tests and for the call after a test function,
    \nullptr is passed

    A logging implementation might chose to record the data row name for
    reporting of results from the test for that data row. It should, in such a
    case, clear its record of the name when called with \nullptr.

    \sa enterTestFunction(), leaveTestFunction()
*/
/*!
    \fn void QAbstractTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line)

    This virtual method is called when an event occurs that relates to the
    resolution of the test. The \a type indicates whether this was a pass, a
    fail or a skip, whether a failure was expected, and whether the test being
    run is blacklisted. The \a description may be empty (for a pass) or a
    message describing the nature of the incident. Where the location in code of
    the incident is known, it is indicated by \a file and \a line; otherwise,
    these are \a nullptr and 0, respectively.

    Every logging implementation must implement this method. Note that there are
    circumstances where more than one incident may be reported, in this way, for
    a single run of a test on a single dataset. It is the implementation's
    responsibility to recognize such cases and decide what to do about them. For
    purposes of counting resolutions of tests in the "Totals" report at the end
    of a test run, QtTest considers the first incident (excluding XFail and its
    blacklisted variant) decisive.

    \sa addMessage(), addBenchmarkResult()
*/
/*!
    \fn void QAbstractTestLogger::addBenchmarkResult(const QBenchmarkResult &result)

    This virtual method is called after a benchmark has been run enough times to
    produce usable data. It is passed the median \a result from all cycles of
    the code controlled by the test's QBENCHMARK loop.

    Every logging implementation must implement this method.

    \sa addIncident(), addMessage()
*/
/*!
    \overload
    \fn void QAbstractTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)

    This virtual method is called, via its \c QtMsgType overload, from the
    custom message handler QtTest installs. It is also used to
    warn about various situations detected by QtTest itself, such
    as \e failure to see a message anticipated by QTest::ignoreMessage() and,
    particularly when verbosity options have been enabled via the command-line,
    to log some extra information.

    Every logging implementation must implement this method. The \a type
    indicates the category of message and the \a message is the content to be
    reported. When the message is associated with specific code, the name of the
    \a file and \a line number within it are also supplied; otherwise, these are
    \nullptr and 0, respectively.

    \sa QTest::ignoreMessage(), addIncident()
*/

/*!
    \overload

    This virtual method is called from the custom message handler QtTest
    installs in place of Qt's default message handler for the duration of
    testing, unless QTest::ignoreMessage() was used to ignore it, or too many
    messages have previously been processed. (The limiting number of messages is
    controlled by the -maxwarnings option to a test and defaults to 2002.)

    Logging implementations should not normally need to override this method.
    The base implementation converts \a type to the matching \l MessageType,
    formats the given \a message suitably for the specified \a context, and
    forwards the converted type and formatted message to the overload that takes
    MessageType and QString.

    \sa QTest::ignoreMessage(), addIncident()
*/
void QAbstractTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context,
                                     const QString &message)
{
    QAbstractTestLogger::MessageTypes messageType = [=]() {
        switch (type) {
        case QtDebugMsg: return QAbstractTestLogger::QDebug;
        case QtInfoMsg: return QAbstractTestLogger::QInfo;
        case QtCriticalMsg: return QAbstractTestLogger::QCritical;
        case QtWarningMsg: return QAbstractTestLogger::QWarning;
        case QtFatalMsg: return QAbstractTestLogger::QFatal;
        }
        Q_UNREACHABLE();
        return QAbstractTestLogger::QFatal;
    }();

    QString formattedMessage = qFormatLogMessage(type, context, message);

    // Note that we explicitly ignore the file and line of the context here,
    // as that's what QTest::messageHandler used to do when calling the same
    // overload directly.
    addMessage(messageType, formattedMessage);
}

namespace QTest
{

/*!
    \fn int QTest::qt_asprintf(QTestCharBuffer *buf, const char *format, ...);
    \internal
 */
int qt_asprintf(QTestCharBuffer *str, const char *format, ...)
{
    constexpr int MAXSIZE = 1024 * 1024 * 2;
    Q_ASSERT(str);
    int size = str->size();
    Q_ASSERT(size > 0);

    va_list ap;
    int res = 0;

    do {
        va_start(ap, format);
        res = qvsnprintf(str->data(), size, format, ap);
        va_end(ap);
        // vsnprintf() reliably '\0'-terminates
        Q_ASSERT(res < 0 || str->data()[res < size ? res : size - 1] == '\0');
        // Note, we're assuming that a result of -1 is always due to running out of space.
        if (res >= 0 && res < size) // Success
            break;

        // Buffer wasn't big enough, try again:
        size *= 2;
        // If too large or out of memory, take what we have:
    } while (size <= MAXSIZE && str->reset(size));

    return res;
}

}

namespace QTestPrivate
{

void generateTestIdentifier(QTestCharBuffer *identifier, int parts)
{
    const char *testObject = parts & TestObject ? QTestResult::currentTestObjectName() : "";
    const char *testFunction = parts & TestFunction ? (QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc") : "";
    const char *objectFunctionFiller = parts & TestObject && parts & (TestFunction | TestDataTag) ? "::" : "";
    const char *testFuctionStart = parts & TestFunction ? "(" : "";
    const char *testFuctionEnd = parts & TestFunction ? ")" : "";

    const char *dataTag = (parts & TestDataTag) && QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
    const char *globalDataTag = (parts & TestDataTag) && QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : "";
    const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : "";

    QTest::qt_asprintf(identifier, "%s%s%s%s%s%s%s%s",
        testObject, objectFunctionFiller, testFunction, testFuctionStart,
        globalDataTag, tagFiller, dataTag, testFuctionEnd);
}

}

QT_END_NAMESPACE