summaryrefslogtreecommitdiffstats
path: root/tests/auto/testlib/selftests/tst_selftests.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp')
-rw-r--r--tests/auto/testlib/selftests/tst_selftests.cpp77
1 files changed, 72 insertions, 5 deletions
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index e332f01861..8167a96eaa 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -68,6 +68,7 @@ private:
QList<LoggerSet> allLoggerSets() const;
QTemporaryDir tempDir;
+ QRegularExpression durationRegExp;
};
struct BenchmarkResult
@@ -99,13 +100,13 @@ inline bool qCompare
// Now check the value. Some variance is allowed, and how much depends on
// the measured unit.
qreal variance = 0.;
- if (r1.unit == "msec") {
+ if (r1.unit == "msecs" || r1.unit == "WalltimeMilliseconds") {
variance = 0.1;
}
else if (r1.unit == "instruction reads") {
variance = 0.001;
}
- else if (r1.unit == "ticks") {
+ else if (r1.unit == "CPU ticks" || r1.unit == "CPUTicks") {
variance = 0.001;
}
if (variance == 0.) {
@@ -233,6 +234,12 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
QStringList() << "lightxml",
QStringList() << "-lightxml" << "-o" << logName("lightxml")
)
+ << LoggerSet("old stdout csv", // benchmarks only
+ QStringList() << "stdout csv",
+ QStringList() << "-csv")
+ << LoggerSet("old csv", // benchmarks only
+ QStringList() << "csv",
+ QStringList() << "-csv" << "-o" << logName("csv"))
// Test with new-style options for a single logger
<< LoggerSet("new stdout txt",
QStringList() << "stdout txt",
@@ -266,6 +273,12 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
QStringList() << "lightxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
)
+ << LoggerSet("new stdout csv", // benchmarks only
+ QStringList() << "stdout csv",
+ QStringList() << "-o" << "-,csv")
+ << LoggerSet("new csv", // benchmarks only
+ QStringList() << "csv",
+ QStringList() << "-o" << logName("csv")+",csv")
// Test with two loggers (don't test all 32 combinations, just a sample)
<< LoggerSet("stdout txt + txt",
QStringList() << "stdout txt" << "txt",
@@ -287,7 +300,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
QStringList() << "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,xunitxml"
)
- // All loggers at the same time
+ // All loggers at the same time (except csv)
<< LoggerSet("all loggers",
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml",
QStringList() << "-o" << logName("txt")+",txt"
@@ -301,6 +314,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
tst_Selftests::tst_Selftests()
: tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX")
+ , durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>")
{}
void tst_Selftests::initTestCase()
@@ -381,6 +395,10 @@ void tst_Selftests::runSubTest_data()
<< "subtest"
<< "verbose1"
<< "verbose2"
+#ifndef QT_NO_EXCEPTIONS
+ // this test will test nothing if the exceptions are disabled
+ << "verifyexceptionthrown"
+#endif //!QT_NO_EXCEPTIONS
<< "warnings"
<< "xunit"
;
@@ -480,6 +498,12 @@ void tst_Selftests::runSubTest_data()
continue;
}
}
+ if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt")))
+ continue; // XML only, do not mix txt and XML for encoding test.
+
+ if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib"))
+ continue;
+
const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
|| subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
|| subtest == QLatin1String("crashes") || subtest == QLatin1String("silent");
@@ -633,9 +657,19 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
.arg(loggers.at(n))));
}
} else {
- QVERIFY2(res.count() == exp.count(),
+ if (res.count() != exp.count()) {
+ qDebug() << "<<<<<<";
+ foreach (const QByteArray &line, res)
+ qDebug() << line;
+ qDebug() << "======";
+ foreach (const QByteArray &line, exp)
+ qDebug() << line;
+ qDebug() << ">>>>>>";
+
+ QVERIFY2(res.count() == exp.count(),
qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3).")
.arg(res.count()).arg(exp.count()).arg(loggers.at(n))));
+ }
}
// By this point, we should have loaded a non-empty expected data file.
@@ -688,7 +722,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
else if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output)
// On some platforms we compile without RTTI, and as a result we never throw an exception.
QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());
- else if (benchmark || line.startsWith("<BenchmarkResult")) {
+ else if (benchmark || line.startsWith("<BenchmarkResult") || (logFormat(logger) == "csv" && line.startsWith('"'))) {
// Don't do a literal comparison for benchmark results, since
// results have some natural variance.
QString error;
@@ -700,6 +734,10 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
QVERIFY2(error.isEmpty(), qPrintable(QString("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(expected)));
QCOMPARE(actualResult, expectedResult);
+ } else if (line.startsWith(" <Duration msecs=") || line.startsWith("<Duration msecs=")) {
+ QRegularExpressionMatch match = durationRegExp.match(line);
+ QVERIFY2(match.hasMatch(), qPrintable(QString::fromLatin1("Invalid Duration tag at line %1 (%2): '%3'")
+ .arg(i).arg(loggers.at(n), output)));
} else {
QVERIFY2(output == expected,
qPrintable(QString::fromLatin1("Mismatch at line %1 (%2): '%3' != '%4'")
@@ -791,6 +829,35 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
out.iterations = iterations;
return out;
}
+
+ if (line.startsWith('"')) {
+ // CSV result
+ // format:
+ // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
+ QStringList split = line.split(',');
+ if (split.count() != 6) {
+ if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
+ return out;
+ }
+
+ bool ok;
+ double total = split.at(4).toDouble(&ok);
+ if (!ok) {
+ if (error) *error = split.at(4) + " is not a valid number";
+ return out;
+ }
+ double iterations = split.at(5).toDouble(&ok);
+ if (!ok) {
+ if (error) *error = split.at(5) + " is not a valid number";
+ return out;
+ }
+
+ out.unit = split.at(2);
+ out.total = total;
+ out.iterations = iterations;
+ return out;
+ }
+
// Text result
// This code avoids using a QRegExp because QRegExp might be broken.
// Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-12 13:33:43 +0000