path: root/tests/auto/testlib/selftests/tst_selftests.cpp

/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Copyright (C) 2016 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the test suite of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include <QtCore/QCoreApplication>

#if QT_CONFIG(process)

#if QT_CONFIG(temporaryfile)
#  define USE_DIFF
#  include <QtCore/QTemporaryFile>
#  include <QtCore/QStandardPaths>
#endif

#include <QtCore/QXmlStreamReader>
#include <QtCore/QFileInfo>
#include <QtCore/QDir>
#include <QtCore/QTemporaryDir>

#include <QtTest/QtTest>

#include <private/cycle_p.h>

#include "emulationdetector.h"

struct BenchmarkResult
{
    qint64  total;
    qint64  iterations;
    QString unit;

    inline QString toString() const
    { return QString("total:%1, unit:%2, iterations:%3").arg(total).arg(unit).arg(iterations); }

    static BenchmarkResult parse(QString const&, QString*);
};

static QString msgMismatch(const QString &actual, const QString &expected)
{
    return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
        + expected + QLatin1Char('\'');
}

static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
                                   QString *errorMessage)
{
    // First make sure the iterations and unit match.
    if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
        // Nope - compare whole string for best failure message
        *errorMessage = msgMismatch(r1.toString(), r2.toString());
        return false;
    }

    // Now check the value.  Some variance is allowed, and how much depends on
    // the measured unit.
    qreal variance = 0.;
    if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
        variance = 0.1;
    else if (r1.unit == QLatin1String("instruction reads"))
        variance = 0.001;
    else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
        variance = 0.001;

    if (variance == 0.) {
        // No variance allowed - compare whole string
        const QString r1S = r1.toString();
        const QString r2S = r2.toString();
        if (r1S != r2S) {
            *errorMessage = msgMismatch(r1S, r2S);
            return false;
        }
        return true;
    }

    if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
        // Whoops, didn't match.  Compare the whole string for the most useful failure message.
        *errorMessage = msgMismatch(r1.toString(), r2.toString());
        return false;
    }
    return true;
}

// Split the passed block of text into an array of lines, replacing any
// filenames and line numbers with generic markers to avoid failing the test
// due to compiler-specific behaviour.
static QList<QByteArray> splitLines(QByteArray ba)
{
    ba.replace('\r', "");
    QList<QByteArray> out = ba.split('\n');

    // Replace any ` file="..."' or ` line="..."'  in XML with a generic location.
    static const char *markers[][2] = {
        { " file=\"", " file=\"__FILE__\"" },
        { " line=\"", " line=\"__LINE__\"" }
    };
    static const int markerCount = sizeof markers / sizeof markers[0];

    for (int i = 0; i < out.size(); ++i) {
        QByteArray& line = out[i];
        for (int j = 0; j < markerCount; ++j) {
            int index = line.indexOf(markers[j][0]);
            if (index == -1) {
                continue;
            }
            const int end = line.indexOf('"', index + int(strlen(markers[j][0])));
            if (end == -1) {
                continue;
            }
            line.replace(index, end-index + 1, markers[j][1]);
        }
    }

    return out;
}

// Helpers for running the 'diff' tool in case comparison fails
#ifdef USE_DIFF
static inline void writeLines(QIODevice &d, const QByteArrayList &lines)
{
    for (const QByteArray &l : lines) {
        d.write(l);
        d.write("\n");
    }
}
#endif // USE_DIFF

static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &actual)
{
    QByteArray result;
#ifdef USE_DIFF
#  ifndef Q_OS_WIN
    const QString diff = QStandardPaths::findExecutable("diff");
#  else
    const QString diff = QStandardPaths::findExecutable("diff.exe");
#  endif
    if (diff.isEmpty())
        return result;
    QTemporaryFile expectedFile;
    if (!expectedFile.open())
        return result;
    writeLines(expectedFile, expected);
    expectedFile.close();
    QTemporaryFile actualFile;
    if (!actualFile.open())
        return result;
    writeLines(actualFile, actual);
    actualFile.close();
    QProcess diffProcess;
    diffProcess.start(diff, {QLatin1String("-u"), expectedFile.fileName(), actualFile.fileName()});
    if (!diffProcess.waitForStarted())
        return result;
    if (diffProcess.waitForFinished())
        result = diffProcess.readAllStandardOutput();
    else
        diffProcess.kill();
#endif // USE_DIFF
    return result;
}

static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }

// Forward declarations
bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
    const QString &actualLine, const QString &expectedLine, QString *errorMessage);
bool checkXml(const QString &logger, QByteArray xml, QString *errorMessage);

bool compareOutput(const QString &logger, const QString &subdir,
                                  const QByteArray &rawOutput, const QByteArrayList &actual,
                                  const QByteArrayList &expected,
                                  QString *errorMessage)
{

    if (actual.size() != expected.size()) {
        *errorMessage = QString::fromLatin1("Mismatch in line count. Expected %1 but got %2.")
                        .arg(expected.size()).arg(actual.size());
        return false;
    }

    // For xml output formats, verify that the log is valid XML.
    if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
        return false;

    // Verify that the actual output is an acceptable match for the
    // expected output.

    const QString qtVersion = QLatin1String(QT_VERSION_STR);
    bool benchmark = false;
    for (int i = 0, size = actual.size(); i < size; ++i) {
        const QByteArray &actualLineBA = actual.at(i);
        // the __FILE__ __LINE__ output is compiler dependent, skip it
        if (actualLineBA.startsWith("   Loc: [") && actualLineBA.endsWith(")]"))
            continue;
        if (actualLineBA.endsWith(" : failure location"))
            continue;

        if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string
            || actualLineBA.startsWith("    <QtBuild") // XML, Light XML build string
            || (actualLineBA.startsWith("    <property name=\"QtBuild\" value="))) { // JUnit-XML build string
            continue;
        }

        QString actualLine = QString::fromLatin1(actualLineBA);
        QString expectedLine = QString::fromLatin1(expected.at(i));
        expectedLine.replace(qtVersionPlaceHolder(), qtVersion);

        if (logger.endsWith(QLatin1String("junitxml"))) {
            static QRegularExpression timestampRegex("timestamp=\".*?\"");
            actualLine.replace(timestampRegex, "timestamp=\"@TEST_START_TIME@\"");
            static QRegularExpression timeRegex("time=\".*?\"");
            actualLine.replace(timeRegex, "time=\"@TEST_DURATION@\"");
        }

        // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
        if (logger.endsWith(QLatin1String("teamcity"))) {
            static QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]");
            actualLine.replace(teamcityLocRegExp, teamCityLocation());
            expectedLine.replace(teamcityLocRegExp, teamCityLocation());
        }

        if (logger.endsWith(QLatin1String("tap"))) {
            if (expectedLine.contains(QLatin1String("at:"))
                || expectedLine.contains(QLatin1String("file:"))
                || expectedLine.contains(QLatin1String("line:")))
                actualLine = expectedLine;
        }

        if (!compareLine(logger, subdir, benchmark, actualLine,
                         expectedLine, errorMessage)) {
            errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)
                                  + QLatin1String(": "));
            return false;
        }

        benchmark = actualLineBA.startsWith("RESULT : ");
    }
    return true;
}

bool compareLine(const QString &logger, const QString &subdir,
                                bool benchmark,
                                const QString &actualLine, const QString &expectedLine,
                                QString *errorMessage)
{
    if (actualLine == expectedLine)
        return true;

    if ((subdir == QLatin1String("assert")
         || subdir == QLatin1String("faildatatype") || subdir == QLatin1String("failfetchtype"))
        && actualLine.contains(QLatin1String("ASSERT: "))
        && expectedLine.contains(QLatin1String("ASSERT: "))) {
        // Q_ASSERT uses __FILE__, the exact contents of which are
        // undefined. If have we something that looks like a Q_ASSERT and we
        // were expecting to see a Q_ASSERT, we'll skip the line.
        return true;
    }

    if (expectedLine.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce"))) {
        // On some platforms we compile without RTTI, and as a result we never throw an exception
        if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
            *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine);
            return false;
        }
        return true;
    }

    if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult"))
        || (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) {
        // Don't do a literal comparison for benchmark results, since
        // results have some natural variance.
        QString error;
        BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
        if (!error.isEmpty()) {
            *errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine);
            return false;
        }
        BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
        if (!error.isEmpty()) {
            *errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine);
            return false;
        }
        return compareBenchmarkResult(actualResult, expectedResult, errorMessage);
    }

    if (actualLine.startsWith(QLatin1String("    <Duration msecs="))
        || actualLine.startsWith(QLatin1String("<Duration msecs="))) {
        static QRegularExpression durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>");
        QRegularExpressionMatch match = durationRegExp.match(actualLine);
        if (match.hasMatch())
            return true;
        *errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine);
        return false;
    }

    if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:")))
        return true;

    const QLatin1String pointerPlaceholder("_POINTER_");
    if (expectedLine.contains(pointerPlaceholder)
        && (expectedLine.contains(QLatin1String("Signal: "))
            || expectedLine.contains(QLatin1String("Slot: ")))) {
        QString actual = actualLine;
        // We don't care about the pointer of the object to whom the signal belongs, so we
        // replace it with _POINTER_, e.g.:
        // Signal: SignalSlotClass(7ffd72245410) signalWithoutParameters ()
        // Signal: QThread(7ffd72245410) started ()
        // After this instance pointer we may have further pointers and
        // references (with an @ prefix) as parameters of the signal or
        // slot being invoked.
        // Signal: SignalSlotClass(_POINTER_) qStringRefSignal ((QString&)@55f5fbb8dd40)
        actual.replace(QRegularExpression("\\b[a-f0-9]{8,}\\b"), pointerPlaceholder);
        // Also change QEventDispatcher{Glib,Win32,etc.} to QEventDispatcherPlatform
        actual.replace(QRegularExpression("\\b(QEventDispatcher)\\w+\\b"), QLatin1String("\\1Platform"));
        if (actual != expectedLine) {
          *errorMessage = msgMismatch(actual, expectedLine);
          return false;
        }
        return true;
    }

    if (EmulationDetector::isRunningArmOnX86() && subdir == QLatin1String("float")) {
        // QEMU cheats at qfloat16, so outputs it as if it were a float.
        if (actualLine.endsWith(QLatin1String("Actual   (operandLeft) : 0.001"))
            && expectedLine.endsWith(QLatin1String("Actual   (operandLeft) : 0.000999"))) {
            return true;
        }
    }

    *errorMessage = msgMismatch(actualLine, expectedLine);
    return false;
}

bool checkXml(const QString &logger, QByteArray xml, QString *errorMessage)
{
    // lightxml intentionally skips the root element, which technically makes it
    // not valid XML.
    // We'll add that ourselves for the purpose of validation.
    if (logger.endsWith(QLatin1String("lightxml"))) {
        xml.prepend("<root>");
        xml.append("</root>");
    }

    QXmlStreamReader reader(xml);
    while (!reader.atEnd())
        reader.readNext();

    if (reader.hasError()) {
        const int lineNumber = int(reader.lineNumber());
        const QByteArray line = xml.split('\n').value(lineNumber - 1);
        *errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4")
                        .arg(lineNumber).arg(reader.columnNumber())
                        .arg(QString::fromLatin1(line), reader.errorString());
        return false;
    }
    return true;
}

// attribute must contain ="
QString extractXmlAttribute(const QString &line, const char *attribute)
{
    int index = line.indexOf(attribute);
    if (index == -1)
        return QString();
    const int attributeLength = int(strlen(attribute));
    const int end = line.indexOf('"', index + attributeLength);
    if (end == -1)
        return QString();

    const QString result = line.mid(index + attributeLength, end - index - attributeLength);
    if (result.isEmpty())
        return ""; // ensure empty but not null
    return result;
}

// Parse line into the BenchmarkResult it represents.
BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
{
    if (error) *error = QString();
    BenchmarkResult out;

    QString remaining = line.trimmed();

    if (remaining.isEmpty()) {
        if (error) *error = "Line is empty";
        return out;
    }

    if (line.startsWith("<BenchmarkResult ")) {
        // XML result
        // format:
        //   <BenchmarkResult metric="$unit" tag="$tag" value="$total" iterations="$iterations" />
        if (!line.endsWith("/>")) {
            if (error) *error = "unterminated XML";
            return out;
        }

        QString unit = extractXmlAttribute(line, " metric=\"");
        QString sTotal = extractXmlAttribute(line, " value=\"");
        QString sIterations = extractXmlAttribute(line, " iterations=\"");
        if (unit.isNull() || sTotal.isNull() || sIterations.isNull()) {
            if (error) *error = "XML snippet did not contain all required values";
            return out;
        }

        bool ok;
        double total = sTotal.toDouble(&ok);
        if (!ok) {
            if (error) *error = sTotal + " is not a valid number";
            return out;
        }
        double iterations = sIterations.toDouble(&ok);
        if (!ok) {
            if (error) *error = sIterations + " is not a valid number";
            return out;
        }

        out.unit = unit;
        out.total = total;
        out.iterations = iterations;
        return out;
    }

    if (line.startsWith('"')) {
        // CSV result
        // format:
        //  "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
        QStringList split = line.split(',');
        if (split.count() != 6) {
            if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
            return out;
        }

        bool ok;
        double total = split.at(4).toDouble(&ok);
        if (!ok) {
            if (error) *error = split.at(4) + " is not a valid number";
            return out;
        }
        double iterations = split.at(5).toDouble(&ok);
        if (!ok) {
            if (error) *error = split.at(5) + " is not a valid number";
            return out;
        }

        out.unit = split.at(2);
        out.total = total;
        out.iterations = iterations;
        return out;
    }

    // Text result
    // This code avoids using a QRegExp because QRegExp might be broken.
    // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)

    QString sFirstNumber;
    while (!remaining.isEmpty() && !remaining.at(0).isSpace()) {
        sFirstNumber += remaining.at(0);
        remaining.remove(0,1);
    }
    remaining = remaining.trimmed();

    // 4,000 -> 4000
    sFirstNumber.remove(',');

    // Should now be parseable as floating point
    bool ok;
    double firstNumber = sFirstNumber.toDouble(&ok);
    if (!ok) {
        if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";
        return out;
    }

    // Remaining: msec per iteration (total: 4000, iterations: 1)
    static const char periterbit[] = " per iteration (total: ";
    QString unit;
    while (!remaining.startsWith(periterbit) && !remaining.isEmpty()) {
        unit += remaining.at(0);
        remaining.remove(0,1);
    }
    if (remaining.isEmpty()) {
        if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";
        return out;
    }

    remaining = remaining.mid(sizeof(periterbit)-1);

    // Remaining: 4,000, iterations: 1)
    static const char itersbit[] = ", iterations: ";
    QString sTotal;
    while (!remaining.startsWith(itersbit) && !remaining.isEmpty()) {
        sTotal += remaining.at(0);
        remaining.remove(0,1);
    }
    if (remaining.isEmpty()) {
        if (error) *error = "Could not find pattern: '<number>, iterations: '";
        return out;
    }

    remaining = remaining.mid(sizeof(itersbit)-1);

    // 4,000 -> 4000
    sTotal.remove(',');

    double total = sTotal.toDouble(&ok);
    if (!ok) {
        if (error) *error = sTotal + " (total) is not a valid number";
        return out;
    }

    // Remaining: 1)
    QString sIters;
    while (remaining != QLatin1String(")") && !remaining.isEmpty()) {
        sIters += remaining.at(0);
        remaining.remove(0,1);
    }
    if (remaining.isEmpty()) {
        if (error) *error = "Could not find pattern: '<num>)'";
        return out;
    }
    qint64 iters = sIters.toLongLong(&ok);
    if (!ok) {
        if (error) *error = sIters + " (iterations) is not a valid integer";
        return out;
    }

    double calcFirstNumber = double(total)/double(iters);
    if (!qFuzzyCompare(firstNumber, calcFirstNumber)) {
        if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(calcFirstNumber).arg(firstNumber);
        return out;
    }

    out.total = total;
    out.unit = unit;
    out.iterations = iters;
    return out;
}

// ----------------------------------------------------------------------

#include "catch_p.h"
#include <QtTest/private/qtestlog_p.h>

#if defined(Q_OS_MACOS)
#include <QtCore/private/qcore_mac_p.h>
#endif

enum RebaseMode { NoRebase, RebaseMissing, RebaseFailing, RebaseAll };
static RebaseMode rebaseMode = NoRebase;

static QTemporaryDir testOutputDir(QDir::tempPath() + "/tst_selftests.XXXXXX");

enum ArgumentStyle { NewStyleArgument, OldStyleArguments };
enum OutputMode { FileOutput, StdoutOutput };

struct TestLogger
{
    TestLogger(QTestLog::LogMode logger) : logger(logger) {}

    TestLogger(QTestLog::LogMode logger, ArgumentStyle argumentStyle)
        : logger(logger), argumentStyle(argumentStyle) {}
    TestLogger(QTestLog::LogMode logger, OutputMode outputMode)
        : logger(logger), outputMode(outputMode) {}

    TestLogger(QTestLog::LogMode logger, OutputMode outputMode, ArgumentStyle argumentStyle)
        : logger(logger), outputMode(outputMode), argumentStyle(argumentStyle) {}

    QString shortName() const
    {
        if (logger == QTestLog::Plain)
            return "txt";

        auto loggers = QMetaEnum::fromType<QTestLog::LogMode>();
        return QString(loggers.valueToKey(logger)).toLower();
    }

    QString outputFileName(const QString &test) const
    {
        if (outputMode == StdoutOutput)
            return QString();

        return testOutputDir.filePath("output_" + test + "." + shortName());
    }

    QString expectationFileName(const QString &test, int version = 0) const
    {
        auto fileName = "expected_" + test;
        if (version)
            fileName += QString("_%1").arg(version);
        fileName += "." + shortName();
        return fileName;
    }

    QStringList arguments(const QString &test) const
    {
        auto fileName = outputFileName(test);

        QStringList arguments;
        if (argumentStyle == NewStyleArgument) {
            arguments << "-o" << (!fileName.isEmpty() ? fileName : QStringLiteral("-"))
                + "," + shortName();
        } else {
            arguments << "-" + shortName();
            if (!fileName.isEmpty())
                arguments << "-o" << fileName;
        }

        return arguments;
    }

    QByteArray testOutput(const QString &test) const
    {
        if (outputMode == StdoutOutput)
            return QByteArray();

        QFile outputFile(outputFileName(test));
        REQUIRE(outputFile.exists());
        REQUIRE(outputFile.open(QIODevice::ReadOnly));
        return outputFile.readAll();
    }

    bool shouldIgnoreTest(const QString &test) const;

    operator QTestLog::LogMode() const { return logger; }

    QTestLog::LogMode logger;
    OutputMode outputMode = FileOutput;
    ArgumentStyle argumentStyle = NewStyleArgument;
};

bool TestLogger::shouldIgnoreTest(const QString &test) const
{
#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
    if (logger == QTestLog::Apple)
        return true;
#endif

    if (test == "deleteLater" || test == "deleteLater_noApp" || test == "mouse")
        return true; // Missing expectation files

    // These tests are affected by timing and whether the CPU tick counter
    // is monotonically increasing. They won't work on some machines so
    // leave them off by default. Feel free to enable them for your own
    // testing by setting the QTEST_ENABLE_EXTRA_SELFTESTS environment
    // variable to something non-empty.
    static bool enableExtraTests = !qEnvironmentVariableIsEmpty("QTEST_ENABLE_EXTRA_SELFTESTS");
    if (!enableExtraTests && (test == "benchlibtickcounter" || test == "benchlibwalltime"))
        return true;

#if defined(Q_OS_WIN)
    // On windows, assert does nothing in release mode and blocks execution
    // with a popup window in debug mode, so skip tests that assert.
    if (test == "assert" || test == "faildatatype" || test == "failfetchtype"
        || test == "fetchbogus")
        return true;
#endif

#if defined(QT_NO_EXCEPTIONS) || defined(Q_CC_INTEL) || defined(Q_OS_WIN)
    // Disable this test on Windows or for Intel compiler, as the run-times
    // will popup dialogs with warnings that uncaught exceptions were thrown
    if (test == "exceptionthrow")
        return true;
#endif

#if defined(QT_NO_EXCEPTIONS)
    // This test will test nothing if the exceptions are disabled
    if (test == "verifyexceptionthrown")
        return true;
#endif

    if (test == "benchlibcallgrind") {
#if !(defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX))
        // Skip on platforms where callgrind is not available
        return true;
#else
        // Check that it's actually available
        QProcess checkProcess;
        QStringList args;
        args << "--version";
        checkProcess.start("valgrind", args);
        if (!checkProcess.waitForFinished(-1))
            WARN("Valgrind broken or not available. Not running benchlibcallgrind test!");
#endif
    }

    if (logger != QTestLog::Plain || outputMode == FileOutput) {
        // The following tests only work with plain text output to stdout,
        // either because they execute multiple test objects or because
        // they internally supply arguments to themselves.
        if (test == "differentexec"
            || test == "multiexec"
            || test == "qexecstringlist"
            || test == "benchliboptions"
            || test == "printdatatags"
            || test == "printdatatagswithglobaltags"
            || test == "silent")
            return true;

        // `crashes' will not output valid XML on platforms without a crash handler
        if (test == "crashes")
            return true;

        // this test prints out some floats in the testlog and the formatting is
        // platform-specific and hard to predict.
        if (test == "float")
            return true;

        // these tests are quite slow, and running them for all the loggers significantly
        // increases the overall test time.  They do not really relate to logging, so it
        // should be safe to run them just for the stdout loggers.
        if (test == "benchlibcallgrind" || test == "sleep")
            return true;
    }

    if (test == "badxml" && !(logger == QTestLog::XML
            || logger == QTestLog::LightXML || logger == QTestLog::JUnitXML))
        return true;

    if (logger == QTestLog::CSV && !test.startsWith("benchlib"))
        return true;

    if (logger == QTestLog::TeamCity && test.startsWith("benchlib"))
        return true; // Skip benchmark for TeamCity logger

    return false;
}

using TestLoggers = QList<TestLogger>;

// ----------------------- Output checking -----------------------

/*
    Check that the test doesn't produce any unexpected error output.

    Some tests may output unpredictable strings to stderr, which we'll ignore.

    For instance, uncaught exceptions on Windows might say (depending on Windows
    version and JIT debugger settings):
    "This application has requested the Runtime to terminate it in an unusual way.
    Please contact the application's support team for more information."

    Also, tests which use valgrind may generate warnings if the toolchain is
    newer than the valgrind version, such that valgrind can't understand the
    debug information on the binary.
*/
void checkErrorOutput(const QString &test, const QByteArray &errorOutput)
{
    if (test == "exceptionthrow"
        || test == "cmptest" // QImage comparison requires QGuiApplication
        || test == "fetchbogus"
        || test == "watchdog"
        || test == "xunit"
        || test == "benchlibcallgrind")
        return;

#ifdef Q_CC_MINGW
    if (test == "blacklisted" // calls qFatal()
        || test == "silent") // calls qFatal()
#endif
        return;

#ifdef Q_OS_LINUX
    // QEMU outputs to stderr about uncaught signals
    if (EmulationDetector::isRunningArmOnX86() &&
        (test == "assert"
         || test == "blacklisted"
         || test == "crashes"
         || test == "faildatatype"
         || test == "failfetchtype"
         || test == "silent"
        ))
        return;
#endif

    INFO(errorOutput.toStdString());
    REQUIRE(errorOutput.isEmpty());
}

/*
    Removes any parts of the output that may vary between test runs.
*/
QByteArray sanitizeOutput(const QString &test, const QByteArray &output)
{
    QByteArray actual = output;

    if (test == "crashes") {
#if !defined(Q_OS_WIN)
        // Remove digits of times
        const QByteArray timePattern("Function time:");
        int timePos = actual.indexOf(timePattern);
        if (timePos >= 0) {
            timePos += timePattern.size();
            const int nextLinePos = actual.indexOf('\n', timePos);
            for (int c = (nextLinePos != -1 ? nextLinePos : actual.size()) - 1; c >= timePos; --c) {
                if (actual.at(c) >= '0' && actual.at(c) <= '9')
                    actual.remove(c, 1);
            }
        }
#endif

#if defined(Q_OS_WIN)
        // Remove stack trace which is output to stdout
        const int exceptionLogStart = actual.indexOf("A crash occurred in ");
        if (exceptionLogStart >= 0)
            actual.truncate(exceptionLogStart);
#endif
    }

    return actual;
}

QByteArray readExpectationFile(const QString &fileName)
{
    QFile file(QStringLiteral(":/") + fileName);

    if (!file.exists() && rebaseMode != NoRebase) {
        // Try rebased test results
        file.setFileName(testOutputDir.filePath(fileName));
    }

    if (!file.exists())
        return QByteArray();

    CAPTURE(file.fileName());
    REQUIRE(file.open(QIODevice::ReadOnly));
    return file.readAll();
}

void checkTestOutput(const QString &test, const TestLogger &logger, const QByteArray &testOutput)
{
    REQUIRE(!testOutput.isEmpty());

    QByteArray actual = sanitizeOutput(test, testOutput);
    auto actualLines = splitLines(actual);

    QString outputMessage;
    bool expectationMatched = false;

    QString expectationFileName;

    // Rebases test results if the given mode has been enabled on the command line
    auto rebaseTestResult = [&](RebaseMode mode) {
        if (rebaseMode < mode)
            return false;

        QFile file(testOutputDir.filePath(expectationFileName));
        REQUIRE(file.open(QIODevice::WriteOnly));
        file.write(actual);

        expectationMatched = true;
        return true;
    };

    bool foundExpectionFile = false;
    for (int version = 0; !expectationMatched; ++version) {
        // Look for a test expectation file. Most tests only have a single
        // expectation file, while some have multiple versions that should
        // all be considered before failing the test.
        expectationFileName = logger.expectationFileName(test, version);

        if (rebaseTestResult(RebaseAll))
            break;

        const QByteArray expected = readExpectationFile(expectationFileName);
        if (expected.isEmpty()) {
            if (rebaseTestResult(RebaseMissing))
                break;

            if (!version) {
                // Look for version-specific expectations
                continue;
            } else {
                // No more versions found, and still no match
                assert(!expectationMatched);
                if (!foundExpectionFile)
                    outputMessage += "Could not find any expectation files for subtest '" + test + "'";
                break;
            }
        }

        // Found expected result
        foundExpectionFile = true;
        QString errorMessage;
        auto expectedLines = splitLines(expected);
        if (compareOutput(logger.shortName(), test, actual, actualLines, expectedLines, &errorMessage)) {
            expectationMatched = true;
        } else if (rebaseTestResult(RebaseFailing)) {
            break;
        } else {
            if (!outputMessage.isEmpty())
                outputMessage += "\n\n" + QString('-').repeated(80) + "\n";
            outputMessage += "\n" + errorMessage + "\n";
            outputMessage += "\nExpected (" + expectationFileName + "):\n" + expected;
            outputMessage += "\nActual:\n" + actual;
            const QByteArray diff = runDiff(expectedLines, actualLines);
            if (!diff.isEmpty())
                 outputMessage += "\nDiff:\n" + diff.trimmed();
        }
    }

    INFO(outputMessage.toStdString());
    CHECK(expectationMatched);
}

// ----------------------- Test running -----------------------

static QProcessEnvironment testEnvironment()
{
    static QProcessEnvironment environment;
    if (environment.isEmpty()) {
        const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
        const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH");
        foreach (const QString &key, systemEnvironment.keys()) {
            const bool useVariable = key == "PATH" || key == "QT_QPA_PLATFORM"
#if defined(Q_OS_QNX)
                || key == "GRAPHICS_ROOT" || key == "TZ"
#elif defined(Q_OS_UNIX)
                || key == "HOME" || key == "USER" // Required for X11 on openSUSE
                || key == "QEMU_SET_ENV" || key == "QEMU_LD_PREFIX" // Required for QEMU
#  if !defined(Q_OS_MACOS)
                || key == "DISPLAY" || key == "XAUTHLOCALHOSTNAME"
                || key.startsWith("XDG_")
#  endif // !Q_OS_MACOS
#endif // Q_OS_UNIX
#ifdef __COVERAGESCANNER__
                || key == "QT_TESTCOCOON_ACTIVE"
#endif
                || ( preserveLibPath && (key == "QT_PLUGIN_PATH"
                                        || key == "LD_LIBRARY_PATH"))
                ;
            if (useVariable)
                environment.insert(key, systemEnvironment.value(key));
        }
        // Avoid interference from any qtlogging.ini files, e.g. in /etc/xdg/QtProject/:
        environment.insert("QT_LOGGING_RULES", "*.debug=true;qt.*=false");

#if defined(Q_OS_UNIX)
        // Avoid the warning from QCoreApplication
        environment.insert("LC_ALL", "en_US.UTF-8");
#endif
    }
    return environment;
}

struct TestProcessResult
{
    int exitCode;
    QByteArray standardOutput;
    QByteArray errorOutput;
};

TestProcessResult runTestProcess(const QString &test, const QStringList &arguments)
{
    QProcessEnvironment environment = testEnvironment();

    const bool crashes = test == "assert" || test == "exceptionthrow"
        || test == "fetchbogus" || test == "crashedterminate"
        || test == "faildatatype" || test == "failfetchtype"
        || test == "crashes" || test == "silent"
        || test == "blacklisted" || test == "watchdog";

    if (crashes) {
        environment.insert("QTEST_DISABLE_CORE_DUMP", "1");
        environment.insert("QTEST_DISABLE_STACK_DUMP", "1");
        if (test == "watchdog")
            environment.insert("QTEST_FUNCTION_TIMEOUT", "100");
    }

    QProcess process;
    process.setProcessEnvironment(environment);
    const QString command = test + '/' + test;
    process.start(command, arguments);

    CAPTURE(command);
    INFO(environment.toStringList().join('\n').toStdString());
    CAPTURE(process.errorString());

    REQUIRE(process.waitForStarted());
    REQUIRE(process.waitForFinished());

    if (!crashes)
         REQUIRE(process.exitStatus() == QProcess::NormalExit);

    return { process.exitCode(), process.readAllStandardOutput(), process.readAllStandardError() };
}

/*
    Runs a single test and verifies the output against the expected results.
*/
void runTest(const QString &test, const TestLoggers &requestedLoggers)
{
    TestLoggers loggers;
    for (auto logger : requestedLoggers) {
        if (!logger.shouldIgnoreTest(test))
            loggers += logger;
    }

    if (loggers.isEmpty())
        return;

    QStringList arguments;
    for (auto logger : loggers)
        arguments += logger.arguments(test);

    CAPTURE(test);
    CAPTURE(arguments);

    auto testProcess = runTestProcess(test, arguments);

    checkErrorOutput(test, testProcess.errorOutput);

    for (auto logger : loggers) {
        QByteArray testOutput;
        if (logger.outputMode == StdoutOutput)
            testOutput = testProcess.standardOutput;
        else
            testOutput = logger.testOutput(test);

        checkTestOutput(test, logger, testOutput);
    }
}

/*
    Runs a single test and verifies the output against the expected result.
*/
void runTest(const QString &test, const TestLogger &logger)
{
    runTest(test, TestLoggers{logger});
}

// ----------------------- Catch helpers -----------------------

template <typename T>
class QtMetaEnumGenerator : public Catch::Generators::IGenerator<T>
{
public:
    QtMetaEnumGenerator()
    {
        metaEnum = QMetaEnum::fromType<T>();
        next();
    }

    bool next() override
    {
        current = static_cast<T>(metaEnum.value(++index));
        return index < metaEnum.keyCount();
    }

    const T& get() const override { return current; }

private:
    QMetaEnum metaEnum;
    int index = -1;
    T current;
};

template <typename T>
Catch::Generators::GeneratorWrapper<T> enums()
{
    return Catch::Generators::GeneratorWrapper<T>(
        std::unique_ptr<Catch::Generators::IGenerator<T>>(
            new QtMetaEnumGenerator<T>()));
}

QT_BEGIN_NAMESPACE
template <typename T, typename A = typename std::enable_if<std::is_same<T, std::string>::value == false, void>::type>
std::ostream& operator<<(std::ostream &os, const T &value)
{
    QString output;
    QDebug debug(&output);
    debug.nospace() << value;
    os << output.toStdString();
    return os;
}
QT_END_NAMESPACE

// ----------------------- Test cases -----------------------

static const auto kBaselineTest = "pass";

bool isCommandLineLogger(QTestLog::LogMode logger)
{
#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
    // The Apple logger is internal and never logs to file or stdout
    return logger != QTestLog::Apple;
#else
    Q_UNUSED(logger);
    return true;
#endif
}

bool isGenericCommandLineLogger(QTestLog::LogMode logger)
{
    // The CSV logger is only used for benchmarks
    return isCommandLineLogger(logger) && logger != QTestLog::CSV;
}

TEST_CASE("Loggers support both old and new style arguments")
{
    auto logger = GENERATE(filter(isGenericCommandLineLogger, enums<QTestLog::LogMode>()));

    GIVEN("The " << logger << " logger") {
        auto argumentStyle = GENERATE(OldStyleArguments, NewStyleArgument);
        WHEN("Passing arguments with " <<
            (argumentStyle == NewStyleArgument ? "new" : "old") << " style") {
            runTest(kBaselineTest, TestLogger(logger, argumentStyle));
        }
    }
}

TEST_CASE("Loggers can output to both file and stdout")
{
    auto logger = GENERATE(filter(isGenericCommandLineLogger, enums<QTestLog::LogMode>()));

    GIVEN("The " << logger << " logger") {
        auto outputMode = GENERATE(StdoutOutput, FileOutput);
        WHEN("Directing output to " << (outputMode == FileOutput ? "file" : "stdout")) {
            runTest(kBaselineTest, TestLogger(logger, outputMode));
        }
    }
}

TEST_CASE("Logging to file and stdout at the same time")
{
    auto loggerEnum = QMetaEnum::fromType<QTestLog::LogMode>();
    for (int i = 0; i < loggerEnum.keyCount(); ++i) {
        auto stdoutLogger = QTestLog::LogMode(loggerEnum.value(i));
        if (!isGenericCommandLineLogger(stdoutLogger))
            continue;

        for (int j = 0; j < loggerEnum.keyCount(); ++j) {
            auto fileLogger = QTestLog::LogMode(loggerEnum.value(j));
            if (!isGenericCommandLineLogger(fileLogger))
                continue;

            runTest(kBaselineTest, TestLoggers{
                TestLogger(fileLogger, FileOutput),
                TestLogger(stdoutLogger, StdoutOutput)
            });
        }
    }
}

TEST_CASE("All loggers can be enabled at the same time")
{
    TestLoggers loggers;

    auto loggerEnum = QMetaEnum::fromType<QTestLog::LogMode>();
    for (int i = 0; i < loggerEnum.keyCount(); ++i) {
        auto logger = QTestLog::LogMode(loggerEnum.value(i));
        if (!isGenericCommandLineLogger(logger))
            continue;

        loggers += TestLogger(logger, FileOutput);
    }

    runTest(kBaselineTest, loggers);
}

SCENARIO("Test output of the loggers is as expected")
{
    static QStringList tests = QString(QT_STRINGIFY(SUBPROGRAMS)).split(' ');

    auto logger = GENERATE(filter(isGenericCommandLineLogger, enums<QTestLog::LogMode>()));

    GIVEN("The " << logger << " logger") {
        for (QString test : tests) {
            AND_GIVEN("The " << test << " subtest") {
                runTest(test, TestLogger(logger));
            }
        }
    }
}

#endif // QT_CONFIG(process)

// ----------------------- Entrypoint -----------------------

int main(int argc, char **argv)
{
#if !QT_CONFIG(process)
    return 0;
#else
    std::vector<const char*> args(argv, argv + argc);

    static auto kRebaseArgument = "--rebase";
    auto rebaseArgument = std::find_if(args.begin(), args.end(),
        [=](const char *arg) { return strncmp(arg, kRebaseArgument, 8) == 0; });
    if (rebaseArgument != args.end()) {
        QString mode((*rebaseArgument) + 8);
        if (mode == "=missing")
            rebaseMode = RebaseMissing;
        else if (mode.isEmpty() || mode == "=failing")
            rebaseMode = RebaseFailing;
        else if (mode == "=all")
            rebaseMode = RebaseAll;

        args.erase(rebaseArgument);
        argc = int(args.size());
        argv = const_cast<char**>(&args[0]);
    }

    QCoreApplication app(argc, argv);

    if (!testOutputDir.isValid())
        qFatal("Could not create temp directory: %s", qUtf8Printable(testOutputDir.errorString()));

    // Detect the location of the sub programs
    QString subProgram = "pass/pass";
#if defined(Q_OS_WIN)
    subProgram += ".exe";
#endif
    QString testdataDir = QFINDTESTDATA(subProgram);
    int testDataDirCutoff = testdataDir.lastIndexOf(subProgram);
    testdataDir = testDataDirCutoff > 0 ? testdataDir.left(testDataDirCutoff)
        : QCoreApplication::applicationDirPath();

    // Move into testdata path and execute tests relative to that
    if (!QDir::setCurrent(testdataDir))
        qFatal("Could not chdir to %s", qUtf8Printable(testdataDir));

    auto result = QTestPrivate::catchMain(argc, argv);

    if (result != 0 || rebaseMode != NoRebase) {
        // Note: Ctrl+C won't pass though here, so the test output won't be kept
        qDebug() << "Test outputs left in" << qUtf8Printable(testOutputDir.path());
        testOutputDir.setAutoRemove(false);
    }

    return result;
#endif
}