From bc8f25c7e6ab73441c46ea41362a054843b42ec3 Mon Sep 17 00:00:00 2001 From: Friedemann Kleint Date: Mon, 19 Dec 2011 11:33:07 +0100 Subject: QTestlib: Make QImage comparison more verbose. Introduce a specialization for qCompare(QImage,QImage) that checks isNull, size and format and outputs verbose messages. Check isNull, size similarly for QPixmap. Add an autotest: - Add test to cmptest and make it a GUI application since QImage requires QGuiApplication. - Make testlib/selftests capable of running X11-GUI applications by passing DISPLAY. - Ignore stderr output for cmptest - Add test data Change-Id: I2b29c7822fbeedf2b22c90889739ed7ff859ce92 Reviewed-by: Jason McDonald --- tests/auto/testlib/selftests/cmptest/cmptest.pro | 2 +- .../auto/testlib/selftests/cmptest/tst_cmptest.cpp | 138 +++++++++++++++++++++ .../testlib/selftests/expected_cmptest.lightxml | 72 +++++++++-- tests/auto/testlib/selftests/expected_cmptest.txt | 52 ++++++-- tests/auto/testlib/selftests/expected_cmptest.xml | 72 +++++++++-- .../testlib/selftests/expected_cmptest.xunitxml | 29 ++++- tests/auto/testlib/selftests/tst_selftests.cpp | 28 ++++- 7 files changed, 357 insertions(+), 36 deletions(-) (limited to 'tests/auto/testlib/selftests') diff --git a/tests/auto/testlib/selftests/cmptest/cmptest.pro b/tests/auto/testlib/selftests/cmptest/cmptest.pro index bc72f23696..1da4646023 100644 --- a/tests/auto/testlib/selftests/cmptest/cmptest.pro +++ b/tests/auto/testlib/selftests/cmptest/cmptest.pro @@ -1,5 +1,5 @@ SOURCES += tst_cmptest.cpp -QT = core testlib +QT = core gui testlib mac:CONFIG -= app_bundle CONFIG -= debug_and_release_target diff --git a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp index 6e19d60cbb..f6773cc4c5 100644 --- a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp +++ b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp @@ -42,6 +42,88 @@ #include #include +#include +#include + +/* XPM test data for QPixmap, QImage tests (use drag cursors as example) */ + +static const char * const xpmPixmapData1[] = { +"11 20 3 1", +". c None", +"a c #FFFFFF", +"X c #000000", // X11 cursor is traditionally black +"aa.........", +"aXa........", +"aXXa.......", +"aXXXa......", +"aXXXXa.....", +"aXXXXXa....", +"aXXXXXXa...", +"aXXXXXXXa..", +"aXXXXXXXXa.", +"aXXXXXXXXXa", +"aXXXXXXaaaa", +"aXXXaXXa...", +"aXXaaXXa...", +"aXa..aXXa..", +"aa...aXXa..", +"a.....aXXa.", +"......aXXa.", +".......aXXa", +".......aXXa", +"........aa."}; + +static const char * const xpmPixmapData2[] = { +"11 20 4 1", +". c None", +"a c #FFFFFF", +"b c #0000FF", +"X c #000000", +"aab........", +"aXab.......", +"aXXab......", +"aXXXab.....", +"aXXXXab....", +"aXXXXXab...", +"aXXXXXXab..", +"aXXXXXXXa..", +"aXXXXXXXXa.", +"aXXXXXXXXXa", +"aXXXXXXaaaa", +"aXXXaXXa...", +"aXXaaXXa...", +"aXa..aXXa..", +"aa...aXXa..", +"a.....aXXa.", +"......aXXa.", +".......aXXa", +".......aXXa", +"........aa."}; + +static const char * const xpmPixmapData3[] = { +"20 20 2 1", +" c #000000", +". c #C32D2D", +" ..........", +" ........", +" .......", +" ......", +" ....", +" ..", +" .", +" ", +" ", +". ", +"... ", +"..... ", +"...... ", +"....... ", +"......... ", +"........... ", +"........... ", +"............ ", +"............ ", +"............. "}; class tst_Cmptest: public QObject { @@ -54,6 +136,10 @@ private slots: void compare_tostring_data(); void compareQStringLists(); void compareQStringLists_data(); + void compareQPixmaps(); + void compareQPixmaps_data(); + void compareQImages(); + void compareQImages_data(); }; static bool boolfunc() { return true; } @@ -222,5 +308,57 @@ void tst_Cmptest::compareQStringLists() QCOMPARE(opA, opB); } +void tst_Cmptest::compareQPixmaps_data() +{ + QTest::addColumn("opA"); + QTest::addColumn("opB"); + + const QPixmap pixmap1(xpmPixmapData1); + const QPixmap pixmap2(xpmPixmapData2); + const QPixmap pixmap3(xpmPixmapData3); + + QTest::newRow("both null") << QPixmap() << QPixmap(); + QTest::newRow("one null") << QPixmap() << pixmap1; + QTest::newRow("other null") << pixmap1 << QPixmap(); + QTest::newRow("equal") << pixmap1 << pixmap1; + QTest::newRow("different size") << pixmap1 << pixmap3; + QTest::newRow("different pixels") << pixmap1 << pixmap2; +} + +void tst_Cmptest::compareQPixmaps() +{ + QFETCH(QPixmap, opA); + QFETCH(QPixmap, opB); + + QCOMPARE(opA, opB); +} + +void tst_Cmptest::compareQImages_data() +{ + QTest::addColumn("opA"); + QTest::addColumn("opB"); + + const QImage image1(QPixmap(xpmPixmapData1).toImage()); + const QImage image2(QPixmap(xpmPixmapData2).toImage()); + const QImage image1Indexed = image1.convertToFormat(QImage::Format_Indexed8); + const QImage image3(QPixmap(xpmPixmapData3).toImage()); + + QTest::newRow("both null") << QImage() << QImage(); + QTest::newRow("one null") << QImage() << image1; + QTest::newRow("other null") << image1 << QImage(); + QTest::newRow("equal") << image1 << image1; + QTest::newRow("different size") << image1 << image3; + QTest::newRow("different format") << image1 << image1Indexed; + QTest::newRow("different pixels") << image1 << image2; +} + +void tst_Cmptest::compareQImages() +{ + QFETCH(QImage, opA); + QFETCH(QImage, opB); + + QCOMPARE(opA, opB); +} + QTEST_MAIN(tst_Cmptest) #include "tst_cmptest.moc" diff --git a/tests/auto/testlib/selftests/expected_cmptest.lightxml b/tests/auto/testlib/selftests/expected_cmptest.lightxml index d027dcde11..17f3e6ec3a 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.lightxml +++ b/tests/auto/testlib/selftests/expected_cmptest.lightxml @@ -12,25 +12,25 @@ - + - + - + - + ) @@ -38,37 +38,91 @@ - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_cmptest.txt b/tests/auto/testlib/selftests/expected_cmptest.txt index f7dc5cfed3..36a4995d91 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.txt +++ b/tests/auto/testlib/selftests/expected_cmptest.txt @@ -6,39 +6,71 @@ PASS : tst_Cmptest::compare_pointerfuncs() FAIL! : tst_Cmptest::compare_tostring(int, string) Compared values are not the same Actual (actual): QVariant(int,123) Expected (expected): QVariant(QString,hi) - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)] + Loc: [tst_cmptest.cpp(214)] FAIL! : tst_Cmptest::compare_tostring(null hash, invalid) Compared values are not the same Actual (actual): QVariant(QVariantHash) Expected (expected): QVariant() - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)] + Loc: [tst_cmptest.cpp(214)] FAIL! : tst_Cmptest::compare_tostring(string, null user type) Compared values are not the same Actual (actual): QVariant(QString,A simple string) Expected (expected): QVariant(PhonyClass) - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)] + Loc: [tst_cmptest.cpp(214)] FAIL! : tst_Cmptest::compare_tostring(both non-null user type) Compared values are not the same Actual (actual): QVariant(PhonyClass,) Expected (expected): QVariant(PhonyClass,) - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)] + Loc: [tst_cmptest.cpp(214)] FAIL! : tst_Cmptest::compareQStringLists(last item different) Compared QStringLists differ at index 2. Actual (opA) : 'string3' Expected (opB) : 'DIFFERS' - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)] + Loc: [tst_cmptest.cpp(308)] FAIL! : tst_Cmptest::compareQStringLists(second-last item different) Compared QStringLists differ at index 2. Actual (opA) : 'string3' Expected (opB) : 'DIFFERS' - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)] + Loc: [tst_cmptest.cpp(308)] FAIL! : tst_Cmptest::compareQStringLists(prefix) Compared QStringLists have different sizes. Actual (opA) size : '2' Expected (opB) size: '1' - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)] + Loc: [tst_cmptest.cpp(308)] FAIL! : tst_Cmptest::compareQStringLists(short list second) Compared QStringLists have different sizes. Actual (opA) size : '12' Expected (opB) size: '1' - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)] + Loc: [tst_cmptest.cpp(308)] FAIL! : tst_Cmptest::compareQStringLists(short list first) Compared QStringLists have different sizes. Actual (opA) size : '1' Expected (opB) size: '12' - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)] + Loc: [tst_cmptest.cpp(308)] +FAIL! : tst_Cmptest::compareQPixmaps(one null) Compared QPixmaps differ. + Actual (opA).isNull() : 1 + Expected (opB).isNull(): 0 + Loc: [tst_cmptest.cpp(333)] +FAIL! : tst_Cmptest::compareQPixmaps(other null) Compared QPixmaps differ. + Actual (opA).isNull() : 0 + Expected (opB).isNull(): 1 + Loc: [tst_cmptest.cpp(333)] +FAIL! : tst_Cmptest::compareQPixmaps(different size) Compared QPixmaps differ in size. + Actual (opA) : 11x20 + Expected (opB): 20x20 + Loc: [tst_cmptest.cpp(333)] +FAIL! : tst_Cmptest::compareQPixmaps(different pixels) Compared values are not the same + Loc: [tst_cmptest.cpp(333)] +FAIL! : tst_Cmptest::compareQImages(one null) Compared QImages differ. + Actual (opA).isNull() : 1 + Expected (opB).isNull(): 0 + Loc: [tst_cmptest.cpp(360)] +FAIL! : tst_Cmptest::compareQImages(other null) Compared QImages differ. + Actual (opA).isNull() : 0 + Expected (opB).isNull(): 1 + Loc: [tst_cmptest.cpp(360)] +FAIL! : tst_Cmptest::compareQImages(different size) Compared QImages differ in size. + Actual (opA) : 11x20 + Expected (opB): 20x20 + Loc: [tst_cmptest.cpp(360)] +FAIL! : tst_Cmptest::compareQImages(different format) Compared QImages differ in format. + Actual (opA) : 6 + Expected (opB): 3 + Loc: [tst_cmptest.cpp(360)] +FAIL! : tst_Cmptest::compareQImages(different pixels) Compared values are not the same + Loc: [tst_cmptest.cpp(360)] PASS : tst_Cmptest::cleanupTestCase() -Totals: 4 passed, 9 failed, 0 skipped +Totals: 4 passed, 18 failed, 0 skipped ********* Finished testing of tst_Cmptest ********* diff --git a/tests/auto/testlib/selftests/expected_cmptest.xml b/tests/auto/testlib/selftests/expected_cmptest.xml index 7faa3a979d..aba1ce5edd 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.xml +++ b/tests/auto/testlib/selftests/expected_cmptest.xml @@ -14,25 +14,25 @@ - + - + - + - + ) @@ -40,37 +40,91 @@ - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_cmptest.xunitxml b/tests/auto/testlib/selftests/expected_cmptest.xunitxml index aa765a65e0..00f5f7f480 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.xunitxml +++ b/tests/auto/testlib/selftests/expected_cmptest.xunitxml @@ -1,5 +1,5 @@ - + @@ -38,6 +38,33 @@ Actual (opA) size : '1' Expected (opB) size: '12'" result="fail"/> + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index ea47678248..7825ade183 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -445,7 +445,15 @@ static inline QProcessEnvironment processEnvironment() { QProcessEnvironment result; const QString path = QStringLiteral("PATH"); - result.insert(path, QProcessEnvironment::systemEnvironment().value(path)); + const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment(); + result.insert(path, systemEnvironment.value(path)); + // Preserve DISPLAY for X11 as some tests use QtGui. +#if defined(Q_OS_UNIX) && !defined(Q_OS_MAC) + const QString display = QStringLiteral("DISPLAY"); + const QString displayValue = systemEnvironment.value(display); + if (!displayValue.isEmpty()) + result.insert(display, displayValue); +#endif return result; } @@ -486,6 +494,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge // newer than the valgrind version, such that valgrind can't understand the // debug information on the binary. if (subdir != QLatin1String("exceptionthrow") + && subdir != QLatin1String("cmptest") // QImage comparison requires QGuiApplication && subdir != QLatin1String("fetchbogus") && subdir != QLatin1String("xunit") && subdir != QLatin1String("benchlibcallgrind")) @@ -524,7 +533,9 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge } } } else { - QCOMPARE(res.count(), exp.count()); + QVERIFY2(res.count() == exp.count(), + qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3).") + .arg(res.count()).arg(exp.count()).arg(loggers.at(n)))); } // For xml output formats, verify that the log is valid XML. @@ -585,7 +596,9 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge QCOMPARE(actualResult, expectedResult); } else { - QCOMPARE(output, expected); + QVERIFY2(output == expected, + qPrintable(QString::fromLatin1("Mismatch at line %1 (%2): '%3' != '%4'") + .arg(i).arg(loggers.at(n), output, expected))); } benchmark = line.startsWith("RESULT : "); @@ -758,9 +771,12 @@ void tst_Selftests::cleanup() // Remove the test output files for (int i = 0; i < loggers.count(); ++i) { - QString logFile = logName(loggers[i]); - if (!logFile.isEmpty()) - QVERIFY(QFile::remove(logFile)); + QString logFileName = logName(loggers[i]); + if (!logFileName.isEmpty()) { + QFile logFile(logFileName); + if (logFile.exists()) + QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: ").arg(logFileName, logFile.errorString()))); + } } } -- cgit v1.2.3