Extend and improve QML auto tests

Exposed a new type to QML auto tests that allows to ignore logging
messages.
In addition assets of QML tests will be copied to the build folder
and instead of running the tests in the source folder, they will be
run in the build folder again.

Change-Id: Ief9d34aed1f74e28f15bd5435723bb6afee7d531
Reviewed-by: Dominik Holland <dominik.holland@pelagicore.com>

4 files changed, 92 insertions, 3 deletions

diff --git a/qmake-features/am-qml-testcase.prf b/qmake-features/am-qml-testcase.prf
index 05c663e1..b5568d71 100644
--- a/qmake-features/am-qml-testcase.prf
+++ b/qmake-features/am-qml-testcase.prf
@@ -7,8 +7,8 @@ load(am-config)
 QT_TOOL_NAME = appman-qmltestrunner
 !isEmpty(TEST_FILES): qtAddTargetEnv(COMMAND, QT)
 
-# Execute test in source folder in order to find any assets.
-TESTRUN_CWD = $$_PRO_FILE_PWD_
+# If the test ends up in a different directory, we should cd to that directory.
+TESTRUN_CWD = $$DESTDIR
 
 debug_and_release:debug_and_release_target {
     # But in debug-and-release-target mode we don't want to cd into the debug/release
@@ -72,3 +72,17 @@ for(file, TEST_FILES) {
 }
 check.commands += true;
 OTHER_FILES += $$TEST_FILES
+
+# Copy assets to build folder
+for (d , DIRECTORIES) {
+    win32: do_copydata.commands += $(COPY_DIR) $$shell_path($$_PRO_FILE_PWD_/$${d}) $$shell_path($$OUT_PWD/$${d}) $$escape_expand(\n\t)
+    else: do_copydata.commands += $(COPY_DIR) $$shell_path($$_PRO_FILE_PWD_/$${d}) $$shell_path($$OUT_PWD) $$escape_expand(\n\t)
+}
+for (f , FILES) {
+    do_copydata.commands += $(COPY) $$shell_path($$_PRO_FILE_PWD_/$${f}) $$shell_path($$OUT_PWD/$${f}) $$escape_expand(\n\t)
+}
+
+!equals(_PRO_FILE_PWD_, $$OUT_PWD) {
+    check.depends = do_copydata
+    QMAKE_EXTRA_TARGETS += do_copydata
+}
diff --git a/src/tools/testrunner/testrunner.cpp b/src/tools/testrunner/testrunner.cpp
index 4bdf270a..618d1b0f 100644
--- a/src/tools/testrunner/testrunner.cpp
+++ b/src/tools/testrunner/testrunner.cpp
@@ -46,10 +46,14 @@
 #include <QObject>
 #include <QPointer>
 #include <QQmlEngine>
+#include <QRegExp>
+#include <QRegularExpression>
 
+#include <qlogging.h>
 #include <QtQml/qqmlpropertymap.h>
 #include <QtTest/qtestsystem.h>
 #include <private/quicktestresult_p.h>
+#include <private/qtestlog_p.h>
 
 QT_BEGIN_NAMESPACE
 namespace QTest {
@@ -65,6 +69,7 @@ class QTestRootObject : public QObject
     Q_PROPERTY(bool windowShown READ windowShown NOTIFY windowShownChanged)
     Q_PROPERTY(bool hasTestCase READ hasTestCase WRITE setHasTestCase NOTIFY hasTestCaseChanged)
     Q_PROPERTY(QObject *defined READ defined)
+
 public:
     QTestRootObject(QObject *parent = nullptr)
         : QObject(parent)
@@ -114,6 +119,65 @@ private:
     friend class TestRunner;
 };
 
+class AmTest : public QObject
+{
+    Q_OBJECT
+
+    AmTest() {}
+
+public:
+    enum MsgType { DebugMsg, WarningMsg, CriticalMsg, FatalMsg, InfoMsg, SystemMsg = CriticalMsg };
+    Q_ENUM(MsgType)
+
+    static AmTest *instance();
+
+    Q_INVOKABLE void ignoreMessage(MsgType type, const char* msg);
+    Q_INVOKABLE void ignoreMessage(MsgType type, const QRegExp &expression);
+};
+
+AmTest *AmTest::instance()
+{
+    static QPointer<AmTest> object = new AmTest;
+    if (!object) {
+        qWarning("A new appman test object has been created, the behavior may be compromised");
+        object = new AmTest;
+    }
+    return object;
+}
+
+static QtMsgType convertMsgType(AmTest::MsgType type)
+{
+    QtMsgType ret;
+
+    switch (type) {
+    case AmTest::WarningMsg: ret = QtWarningMsg; break;
+    case AmTest::CriticalMsg: ret = QtCriticalMsg; break;
+    case AmTest::FatalMsg: ret = QtFatalMsg; break;
+    case AmTest::InfoMsg: ret = QtInfoMsg; break;
+    default: ret = QtDebugMsg;
+    }
+    return ret;
+}
+
+void AmTest::ignoreMessage(MsgType type, const char *msg)
+{
+    QTestLog::ignoreMessage(convertMsgType(type), msg);
+}
+
+void AmTest::ignoreMessage(MsgType type, const QRegExp &expression)
+{
+#ifndef QT_NO_REGULAREXPRESSION
+    QRegularExpression re(expression.pattern());
+    if (expression.caseSensitivity() == Qt::CaseInsensitive)
+        re.setPatternOptions(QRegularExpression::CaseInsensitiveOption);
+    QTestLog::ignoreMessage(convertMsgType(type), re);
+#else
+    Q_UNUSED(type);
+    Q_UNUSED(expression);
+    qWarning() << "Cannot ignore message: regular expressions are not supported";
+#endif
+}
+
 static QObject *testRootObject(QQmlEngine *engine, QJSEngine *jsEngine)
 {
     Q_UNUSED(engine);
@@ -121,6 +185,13 @@ static QObject *testRootObject(QQmlEngine *engine, QJSEngine *jsEngine)
     return QTestRootObject::instance();
 }
 
+static QObject *amTest(QQmlEngine *engine, QJSEngine *jsEngine)
+{
+    Q_UNUSED(engine);
+    Q_UNUSED(jsEngine);
+    return AmTest::instance();
+}
+
 void TestRunner::initialize(const QStringList &testRunnerArguments)
 {
     Q_ASSERT(!testRunnerArguments.isEmpty());
@@ -137,8 +208,9 @@ void TestRunner::initialize(const QStringList &testRunnerArguments)
     QuickTestResult::parseArgs(testArgV.size(), testArgV.data());
     qputenv("QT_QTESTLIB_RUNNING", "1");
 
-    // Register the test object
+    // Register the test object and application-manager test add-on
     qmlRegisterSingletonType<QTestRootObject>("Qt.test.qtestroot", 1, 0, "QTestRootObject", testRootObject);
+    qmlRegisterSingletonType<AmTest>("QtApplicationManager", 1, 0, "AmTest", amTest);
 
     QTestRootObject::instance()->init();
 }
diff --git a/tests/qml/installer/installer.pro b/tests/qml/installer/installer.pro
index 64cb6fec..d85f1607 100644
--- a/tests/qml/installer/installer.pro
+++ b/tests/qml/installer/installer.pro
@@ -1,4 +1,6 @@
 AM_CONFIG = am-config.yaml
 TEST_FILES = tst_installer.qml
 
+FILES = appv1.pkg appv2.pkg
+
 load(am-qml-testcase)
diff --git a/tests/qml/windowmapping/tst_windowmapping.qml b/tests/qml/windowmapping/tst_windowmapping.qml
index e639008e..92e2db89 100644
--- a/tests/qml/windowmapping/tst_windowmapping.qml
+++ b/tests/qml/windowmapping/tst_windowmapping.qml
@@ -269,6 +269,7 @@ TestCase {
         appId = "test.winmap.ping";
         if (ApplicationManager.singleProcess)
             skip("Wayland ping-pong is only supported in multi-process mode");
+        AmTest.ignoreMessage(AmTest.CriticalMsg, /Stopping application.*because we did not receive a Wayland-Pong/);
         ApplicationManager.startApplication(appId);
         windowReadySpy.wait(2000);
         compare(ApplicationManager.applicationRunState(appId), ApplicationManager.Running)