QTestlib/selftests: Refactor comparison

The "crashes" subtest has several "expected" reference files. The matching
one was previously determined by checking the line count of the output.

This however does not work when there are several reference files with
identical line count as is now the case with boot2qt.

Refactor the comparison code from the QTest/preliminary void return to
the bool f(QString* errorMessage) convention so that all files can be tried.

While doing so, streamline the code and remove numerous unneeded
allocations of strings and regular expressions per compared line.

Task-number: QTBUG-65845
Change-Id: I722159d1753f2a36f0e497e315ffd81cb58cac0b
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

1 files changed, 206 insertions, 161 deletions

diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 2ef7258d81..a9c5d40734 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -62,11 +62,22 @@ private slots:
 
 private:
     void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes);
+    bool compareOutput(const QString &logger, const QString &subdir,
+                       const QByteArray &rawOutput, const QByteArrayList &actual,
+                       const QByteArrayList &expected,
+                       QString *errorMessage) const;
+    bool compareLine(const QString &logger, const QString &subdir, bool benchmark,
+                     const QString &actualLine, const QString &expLine,
+                     QString *errorMessage) const;
+    bool checkXml(const QString &logger, QByteArray rawOutput,
+                  QString *errorMessage) const;
+
     QString logName(const QString &logger) const;
     QList<LoggerSet> allLoggerSets() const;
 
     QTemporaryDir tempDir;
     QRegularExpression durationRegExp;
+    QRegularExpression teamcityLocRegExp;
 };
 
 struct BenchmarkResult
@@ -81,46 +92,50 @@ struct BenchmarkResult
     static BenchmarkResult parse(QString const&, QString*);
 };
 
-QT_BEGIN_NAMESPACE
-namespace QTest
+static QString msgMismatch(const QString &actual, const QString &expected)
 {
-template <>
-inline bool qCompare
-    (BenchmarkResult const &r1, BenchmarkResult const &r2,
-     const char* actual, const char* expected, const char* file, int line)
+    return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'")
+        + expected + QLatin1Char('\'');
+}
+
+static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2,
+                                   QString *errorMessage)
 {
     // First make sure the iterations and unit match.
     if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
         // Nope - compare whole string for best failure message
-        return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+        *errorMessage = msgMismatch(r1.toString(), r2.toString());
+        return false;
     }
 
     // Now check the value.  Some variance is allowed, and how much depends on
     // the measured unit.
     qreal variance = 0.;
-    if (r1.unit == "msecs" || r1.unit == "WalltimeMilliseconds") {
+    if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds"))
         variance = 0.1;
-    }
-    else if (r1.unit == "instruction reads") {
+    else if (r1.unit == QLatin1String("instruction reads"))
         variance = 0.001;
-    }
-    else if (r1.unit == "CPU ticks" || r1.unit == "CPUTicks") {
+    else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks"))
         variance = 0.001;
-    }
+
     if (variance == 0.) {
         // No variance allowed - compare whole string
-        return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+        const QString r1S = r1.toString();
+        const QString r2S = r2.toString();
+        if (r1S != r2S) {
+            *errorMessage = msgMismatch(r1S, r2S);
+            return false;
+        }
+        return true;
     }
 
-    if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) {
-        return compare_helper(true, 0, 0, 0, actual, expected, file, line);
+    if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) {
+        // Whoops, didn't match.  Compare the whole string for the most useful failure message.
+        *errorMessage = msgMismatch(r1.toString(), r2.toString());
+        return false;
     }
-
-    // Whoops, didn't match.  Compare the whole string for the most useful failure message.
-    return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+    return true;
 }
-}
-QT_END_NAMESPACE
 
 // Split the passed block of text into an array of lines, replacing any
 // filenames and line numbers with generic markers to avoid failing the test
@@ -227,26 +242,6 @@ static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList &
     return result;
 }
 
-// Print the difference preferably using 'diff', else just print lines
-static void printDifference(const QByteArrayList &expected, const QByteArrayList &actual)
-{
-    QDebug info = qInfo();
-    info.noquote();
-    info.nospace();
-    const QByteArray diff = runDiff(expected, actual);
-    if (diff.isEmpty()) {
-        info << "<<<<<<\n";
-        for (const QByteArray &line : actual)
-            info << line << '\n';
-        info << "======\n";
-        for (const QByteArray &line : expected)
-            info << line << '\n';
-        info << ">>>>>>\n";
-    } else {
-        info << diff;
-    }
-}
-
 // Each test is run with a set of one or more test output loggers.
 // This struct holds information about one such test.
 struct LoggerSet
@@ -399,6 +394,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
 tst_Selftests::tst_Selftests()
     : tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
     , durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>")
+    , teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]")
 {}
 
 void tst_Selftests::initTestCase()
@@ -760,147 +756,196 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
         }
 
         QList<QByteArray> res = splitLines(actualOutputs[n]);
-        const QString expectedFileName = expectedFileNameFromTest(subdir, logger);
-        QList<QByteArray> exp = expectedResult(expectedFileName);
+        QString errorMessage;
+        QString expectedFileName = expectedFileNameFromTest(subdir, logger);
+        if (QFileInfo::exists(expectedFileName)) {
+            QByteArrayList exp = expectedResult(expectedFileName);
 #ifdef Q_CC_MINGW
-        // MinGW formats double numbers differently
-        if (n == 0 && subdir == QStringLiteral("float")) {
-            for (int i = 0; i < exp.size(); ++i) {
-                exp[i].replace("e-07", "e-007");
-                exp[i].replace("e+07", "e+007");
-            }
-        }
-#endif
-
-        // For the "crashes" test, there are multiple versions of the
-        // expected output.  Load the one with the same line count as
-        // the actual output.
-        if (exp.count() == 0) {
-            QList<QList<QByteArray> > expArr;
-            QList<QByteArray> tmp;
-            int i = 1;
-            do {
-                tmp = expectedResult(expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i++), logger));
-                if (tmp.count())
-                    expArr += tmp;
-            } while (tmp.count());
-
-            for (int j = 0; j < expArr.count(); ++j) {
-                if (res.count() == expArr.at(j).count()) {
-                    exp = expArr.at(j);
-                    break;
+            // MinGW formats double numbers differently (last verified with 7.1)
+            if (n == 0 && subdir == QStringLiteral("float")) {
+                for (int i = 0; i < exp.size(); ++i) {
+                    exp[i].replace("e-07", "e-007");
+                    exp[i].replace("e+07", "e+007");
                 }
             }
-
-            if (expArr.count()) {
-                QVERIFY2(exp.count(),
-                         qPrintable(QString::fromLatin1("None of the expected output files for "
-                                                        "%1 format has matching line count.")
-                                    .arg(loggers.at(n))));
+#endif
+            if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) {
+                errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ")
+                                     + expectedFileName + QLatin1Char(' '));
+                errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]);
+                const QByteArray diff = runDiff(exp, res);
+                if (!diff.isEmpty())
+                    errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff);
+                QFAIL(qPrintable(errorMessage));
             }
         } else {
-            if (res.count() != exp.count()) {
-                printDifference(exp, res);
-                QVERIFY2(res.count() == exp.count(),
-                     qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3, %4).")
-                                .arg(res.count()).arg(exp.count()).arg(loggers.at(n), expectedFileName)));
+            // For the "crashes" and other tests, there are multiple versions of the
+            // expected output. Loop until a matching one is found.
+            bool ok = false;
+            for (int i = 1; !ok; ++i) {
+                expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger);
+                const QByteArrayList exp = expectedResult(expectedFileName);
+                if (exp.isEmpty())
+                    break;
+                QString errorMessage2;
+                ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2);
+                if (!ok)
+                    errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2;
+            }
+            if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output.
+                errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir);
+                errorMessage += QLatin1String("\nActual:\n");
+                QDebug(&errorMessage) << actualOutputs[n];
+                QFAIL(qPrintable(errorMessage));
             }
         }
+    }
+}
 
-        // By this point, we should have loaded a non-empty expected data file.
-        QVERIFY2(exp.count(),
-                 qPrintable(QString::fromLatin1("Expected test data for %1 format is empty or not found.")
-                            .arg(loggers.at(n))));
-
-        // For xml output formats, verify that the log is valid XML.
-        if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") {
-            QByteArray xml(actualOutputs[n]);
-            // lightxml intentionally skips the root element, which technically makes it
-            // not valid XML.
-            // We'll add that ourselves for the purpose of validation.
-            if (logFormat(logger) == "lightxml") {
-                xml.prepend("<root>");
-                xml.append("</root>");
-            }
+static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); }
+static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); }
 
-            QXmlStreamReader reader(xml);
+bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
+                                  const QByteArray &rawOutput, const QByteArrayList &actual,
+                                  const QByteArrayList &expected,
+                                  QString *errorMessage) const
+{
 
-            while (!reader.atEnd())
-                reader.readNext();
+    if (actual.size() != expected.size()) {
+        *errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.")
+                        .arg(actual.size()).arg(expected.size());
+        return false;
+    }
 
-            QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3")
-                .arg(reader.lineNumber())
-                .arg(reader.columnNumber())
-                .arg(reader.errorString())
-            ));
+    // For xml output formats, verify that the log is valid XML.
+    if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage))
+        return false;
+
+    // Verify that the actual output is an acceptable match for the
+    // expected output.
+
+    const QString qtVersion = QLatin1String(QT_VERSION_STR);
+    bool benchmark = false;
+    for (int i = 0, size = actual.size(); i < size; ++i) {
+        const QByteArray &actualLineBA = actual.at(i);
+        // the __FILE__ __LINE__ output is compiler dependent, skip it
+        if (actualLineBA.startsWith("   Loc: [") && actualLineBA.endsWith(")]"))
+            continue;
+        if (actualLineBA.endsWith(" : failure location"))
+            continue;
+
+        if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string
+            || actualLineBA.startsWith("    <QtBuild") // XML, Light XML build string
+            || (actualLineBA.startsWith("    <property value=") &&  actualLineBA.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
+            continue;
         }
 
-        // Verify that the actual output is an acceptable match for the
-        // expected output.
-        bool benchmark = false;
-        for (int i = 0; i < res.count(); ++i) {
-            QByteArray line = res.at(i);
-            // the __FILE__ __LINE__ output is compiler dependent, skip it
-            if (line.startsWith("   Loc: [") && line.endsWith(")]"))
-                continue;
-            if (line.endsWith(" : failure location"))
-                continue;
+        QString actualLine = QString::fromLatin1(actualLineBA);
+        QString expectedLine = QString::fromLatin1(expected.at(i));
+        expectedLine.replace(qtVersionPlaceHolder(), qtVersion);
 
-            if (line.startsWith("Config: Using QtTest library") // Text build string
-                || line.startsWith("    <QtBuild") // XML, Light XML build string
-                || (line.startsWith("    <property value=") &&  line.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string
-                    continue;
-            }
+        // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
+        if (logger.endsWith(QLatin1String("teamcity"))) {
+            actualLine.replace(teamcityLocRegExp, teamCityLocation());
+            expectedLine.replace(teamcityLocRegExp, teamCityLocation());
+        }
 
-            QByteArray expLine = exp.at(i);
+        if (!compareLine(logger, subdir, benchmark, actualLine,
+                         expectedLine, errorMessage)) {
+            errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)
+                                  + QLatin1String(": "));
+            return false;
+        }
 
-            // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity
-            if (logFormat(logger) == "teamcity") {
-                QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]");
-                line = QString(line).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1();
-                expLine = QString(expLine).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1();
-            }
+        benchmark = actualLineBA.startsWith("RESULT : ");
+    }
+    return true;
+}
 
-            const QString output(QString::fromLatin1(line));
-            const QString expected(QString::fromLatin1(expLine).replace("@INSERT_QT_VERSION_HERE@", QT_VERSION_STR));
+bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
+                                bool benchmark,
+                                const QString &actualLine, const QString &expectedLine,
+                                QString *errorMessage) const
+{
+    if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: "))
+        && expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) {
+        // Q_ASSERT uses __FILE__, the exact contents of which are
+        // undefined. If have we something that looks like a Q_ASSERT and we
+        // were expecting to see a Q_ASSERT, we'll skip the line.
+        return true;
+    }
 
-            if (subdir == "assert" && output.contains("ASSERT: ") && expected.contains("ASSERT: ") && output != expected)
-                // Q_ASSERT uses __FILE__, the exact contents of which are
-                // undefined.  If we something that looks like a Q_ASSERT and we
-                // were expecting to see a Q_ASSERT, we'll skip the line.
-                continue;
-            else if (expected.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce")) && expected != output)
-                // On some platforms we compile without RTTI, and as a result we never throw an exception.
-                QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());
-            else if (benchmark || line.startsWith("<BenchmarkResult") || (logFormat(logger) == "csv" && line.startsWith('"'))) {
-                // Don't do a literal comparison for benchmark results, since
-                // results have some natural variance.
-                QString error;
-
-                BenchmarkResult actualResult = BenchmarkResult::parse(output, &error);
-                QVERIFY2(error.isEmpty(), qPrintable(QString("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(output)));
-
-                BenchmarkResult expectedResult = BenchmarkResult::parse(expected, &error);
-                QVERIFY2(error.isEmpty(), qPrintable(QString("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(expected)));
-
-                QCOMPARE(actualResult, expectedResult);
-            } else if (line.startsWith("    <Duration msecs=") || line.startsWith("<Duration msecs=")) {
-                QRegularExpressionMatch match = durationRegExp.match(line);
-                QVERIFY2(match.hasMatch(), qPrintable(QString::fromLatin1("Invalid Duration tag at line %1 (%2): '%3'")
-                                                      .arg(i).arg(loggers.at(n), output)));
-            } else if (line.startsWith("Totals:")) {
-                const int lastCommaPos = line.lastIndexOf(',');
-                if (lastCommaPos > 0)
-                    line.truncate(lastCommaPos); // Plain text logger: strip time (", 2323dms").
-            } else {
-                QVERIFY2(output == expected,
-                         qPrintable(QString::fromLatin1("Mismatch at line %1 (%2, %3):\n'%4'\n !=\n'%5'")
-                                    .arg(i + 1).arg(loggers.at(n), expectedFileName, output, expected)));
-            }
+    if (expectedLine.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce"))
+        && actualLine != expectedLine) {
+        // On some platforms we compile without RTTI, and as a result we never throw an exception
+        if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) {
+            *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine);
+            return false;
+        }
+        return true;
+    }
 
-            benchmark = line.startsWith("RESULT : ");
+    if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult"))
+        || (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) {
+        // Don't do a literal comparison for benchmark results, since
+        // results have some natural variance.
+        QString error;
+        BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error);
+        if (!error.isEmpty()) {
+            *errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine);
+            return false;
+        }
+        BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error);
+        if (!error.isEmpty()) {
+            *errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine);
+            return false;
         }
+        return compareBenchmarkResult(actualResult, expectedResult, errorMessage);
+    }
+
+    if (actualLine.startsWith(QLatin1String("    <Duration msecs="))
+        || actualLine.startsWith(QLatin1String("<Duration msecs="))) {
+        QRegularExpressionMatch match = durationRegExp.match(actualLine);
+        if (match.hasMatch())
+            return true;
+        *errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine);
+        return false;
+    }
+
+    if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:")))
+        return true;
+
+    if (actualLine == expectedLine)
+        return true;
+
+    *errorMessage = msgMismatch(actualLine, expectedLine);
+    return false;
+}
+
+bool tst_Selftests::checkXml(const QString &logger, QByteArray xml,
+                             QString *errorMessage) const
+{
+    // lightxml intentionally skips the root element, which technically makes it
+    // not valid XML.
+    // We'll add that ourselves for the purpose of validation.
+    if (logger.endsWith(QLatin1String("lightxml"))) {
+        xml.prepend("<root>");
+        xml.append("</root>");
+    }
+
+    QXmlStreamReader reader(xml);
+    while (!reader.atEnd())
+        reader.readNext();
+
+    if (reader.hasError()) {
+        const int lineNumber = int(reader.lineNumber());
+        const QByteArray line = xml.split('\n').value(lineNumber - 1);
+        *errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4")
+                        .arg(lineNumber).arg(reader.columnNumber())
+                        .arg(QString::fromLatin1(line), reader.errorString());
+        return false;
     }
+    return true;
 }
 
 #endif // QT_CONFIG(process)