Use docker-based CoAP test servers for tests

 - Added docker compose files for setting up CoAP test servers.

 - Enabled the disabled tests which require a californium-based CoAP
   server. These tests are enabled only on linux, since docker-based
   testing doesn't work in CI for other platforms. Note, that they
   are blacklisted for some linux platforms for the same reason.

Change-Id: Id99f9f67fb2874576524e6452250be5506bdbdf4
Reviewed-by: Ryan Chu <ryan.chu@qt.io>

12 files changed, 245 insertions, 23 deletions

diff --git a/tests/auto/auto.pro b/tests/auto/auto.pro
index 5bfc66e..32e4ac4 100644
--- a/tests/auto/auto.pro
+++ b/tests/auto/auto.pro
@@ -1,17 +1,15 @@
 TEMPLATE = subdirs
 
-# TODO: enable disabled tests, when CI is configured properly
-
 SUBDIRS += \
     cmake \
-#    qcoapclient \
+    qcoapclient \
     qcoapmessage \
     qcoapoption \
     qcoaprequest \
     qcoapresource
 
 qtConfig(private_tests): SUBDIRS += \
-#    qcoapqudpconnection \
+    qcoapqudpconnection \
     qcoapinternalrequest \
     qcoapinternalreply \
     qcoapreply
diff --git a/tests/auto/coapnetworksettings.h b/tests/auto/coapnetworksettings.h
index 0e1ee04..3dc5534 100644
--- a/tests/auto/coapnetworksettings.h
+++ b/tests/auto/coapnetworksettings.h
@@ -31,6 +31,7 @@
 #include <QtTest>
 #include <QtCore/qstring.h>
 #include <QtNetwork/qhostinfo.h>
+#include <QtCoap/qcoapclient.h>
 
 /*!
     \internal
@@ -49,23 +50,77 @@
 */
 namespace QtCoapNetworkSettings
 {
-    QString testServerHost()
-    {
+
+#if defined(COAP_TEST_SERVER_IP) || defined(QT_TEST_SERVER)
+#define CHECK_FOR_COAP_SERVER
+#else
+#define CHECK_FOR_COAP_SERVER \
+    QSKIP("CoAP server is not setup, skipping the test..."); \
+    return;
+#endif
+
+#if defined(QT_TEST_SERVER) && !defined(COAP_TEST_SERVER_IP)
+static QString tryToResolveHostName(const QString &hostName)
+{
+    const auto hostInfo = QHostInfo::fromName(hostName);
+    if (!hostInfo.addresses().empty())
+        return hostInfo.addresses().first().toString();
+
+    qWarning() << "Could not resolve the hostname"<< hostName;
+    return hostName;
+}
+#endif
+
+static QString getHostAddress(const QString &serverName)
+{
 #if defined(COAP_TEST_SERVER_IP)
-        return QStringLiteral(COAP_TEST_SERVER_IP);
+    Q_UNUSED(serverName);
+    return QStringLiteral(COAP_TEST_SERVER_IP);
+#elif defined(QT_TEST_SERVER_NAME)
+    QString hostname = serverName % "." % QString(QT_TEST_SERVER_DOMAIN);
+    return tryToResolveHostName(hostname);
+#elif defined(QT_TEST_SERVER)
+    Q_UNUSED(serverName);
+    QString hostname = "qt-test-server." % QString(QT_TEST_SERVER_DOMAIN);
+    return tryToResolveHostName(hostname);
 #else
-        static_assert(false, "COAP_TEST_SERVER_IP variable must be set");
+    Q_UNUSED(serverName);
+    qWarning("This test will fail, "
+             "please set the COAP_TEST_SERVER_IP variable to specify the CoAP server.");
+    return "";
 #endif
-    }
+}
 
-    QString testServerUrl()
-    {
-        return QStringLiteral("coap://") + testServerHost() + QStringLiteral(":")
-                + QString::number(QtCoap::DefaultPort);
-    }
+QString testServerHost()
+{
+    static QString testServerHostAddress = getHostAddress("californium");
+    return testServerHostAddress;
+}
 
-    QString testServerResource()
-    {
-        return testServerUrl() + QStringLiteral("/test");
+QString testServerUrl()
+{
+    return QStringLiteral("coap://") + testServerHost() + QStringLiteral(":")
+            + QString::number(QtCoap::DefaultPort);
+}
+
+QString testServerResource()
+{
+    return testServerUrl() + QStringLiteral("/test");
+}
+
+bool waitForHost(const QUrl &url, quint8 retries = 10)
+{
+    while (retries-- > 0) {
+        QCoapClient client;
+        QSignalSpy spyClientFinished(&client, SIGNAL(finished(QCoapReply *)));
+        client.get(url);
+
+        spyClientFinished.wait(1000);
+
+        if (spyClientFinished.count() == 1)
+            return true;
     }
+    return false;
+}
+
 }
diff --git a/tests/auto/coaptestserver.pri b/tests/auto/coaptestserver.pri
index 420aa89..2399826 100644
--- a/tests/auto/coaptestserver.pri
+++ b/tests/auto/coaptestserver.pri
@@ -1,7 +1,5 @@
 COAP_TEST_SERVER_IP = $$(COAP_TEST_SERVER_IP)
-isEmpty( COAP_TEST_SERVER_IP ) {
-    error(No IP set for CoAP plugtest server. Please set COAP_TEST_SERVER_IP environment variable to run this test.)
+!isEmpty(COAP_TEST_SERVER_IP) {
+    DEFINES += COAP_TEST_SERVER_IP=\\\"$${COAP_TEST_SERVER_IP}\\\"
+    message(CoAP plugtest server IP set to $$COAP_TEST_SERVER_IP)
 }
-
-DEFINES += COAP_TEST_SERVER_IP=\\\"$${COAP_TEST_SERVER_IP}\\\"
-message(CoAP plugtest server IP set to $$COAP_TEST_SERVER_IP)
diff --git a/tests/auto/qcoapclient/BLACKLIST b/tests/auto/qcoapclient/BLACKLIST
new file mode 100644
index 0000000..3719b37
--- /dev/null
+++ b/tests/auto/qcoapclient/BLACKLIST
@@ -0,0 +1,3 @@
+rhel ci
+opensuse ci
+opensuse-leap ci
diff --git a/tests/auto/qcoapclient/qcoapclient.pro b/tests/auto/qcoapclient/qcoapclient.pro
index 5c7fe1e..1483994 100644
--- a/tests/auto/qcoapclient/qcoapclient.pro
+++ b/tests/auto/qcoapclient/qcoapclient.pro
@@ -6,3 +6,6 @@ include(../coaptestserver.pri)
 HEADERS += ../coapnetworksettings.h
 
 SOURCES +=  tst_qcoapclient.cpp
+
+CONFIG += unsupported/testserver
+QT_TEST_SERVER_LIST = californium
diff --git a/tests/auto/qcoapclient/tst_qcoapclient.cpp b/tests/auto/qcoapclient/tst_qcoapclient.cpp
index 5ca695c..f938a04 100644
--- a/tests/auto/qcoapclient/tst_qcoapclient.cpp
+++ b/tests/auto/qcoapclient/tst_qcoapclient.cpp
@@ -51,6 +51,7 @@ class tst_QCoapClient : public QObject
     Q_OBJECT
 
 private Q_SLOTS:
+    void initTestCase();
     void incorrectUrls_data();
     void incorrectUrls();
     void methods_data();
@@ -194,6 +195,13 @@ public slots:
     }
 };
 
+void tst_QCoapClient::initTestCase()
+{
+#if defined(COAP_TEST_SERVER_IP) || defined(QT_TEST_SERVER)
+    QVERIFY2(waitForHost(testServerHost()), "Failed to connect to Californium plugtest server.");
+#endif
+}
+
 void tst_QCoapClient::incorrectUrls_data()
 {
     QWARN("Expect warnings here...");
@@ -257,6 +265,8 @@ void tst_QCoapClient::methods_data()
 
 void tst_QCoapClient::methods()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QUrl, url);
     QFETCH(QtCoap::Method, method);
 
@@ -312,6 +322,8 @@ void tst_QCoapClient::methods()
 
 void tst_QCoapClient::separateMethod()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QCoapClient client;
     QScopedPointer<QCoapReply> reply(client.get(QUrl(testServerUrl() + "/separate")));
 
@@ -327,6 +339,8 @@ void tst_QCoapClient::separateMethod()
 
 void tst_QCoapClient::removeReply()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QCoapClient client;
     QCoapReply *reply = client.get(QUrl(testServerResource()));
     QVERIFY2(reply != nullptr, "Request failed unexpectedly");
@@ -384,6 +398,8 @@ void tst_QCoapClient::requestWithQIODevice_data()
 
 void tst_QCoapClient::requestWithQIODevice()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QUrl, url);
 
     QCoapClient client;
@@ -416,6 +432,8 @@ void tst_QCoapClient::requestWithQIODevice()
 
 void tst_QCoapClient::multipleRequests()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QCoapClient client;
     QUrl url = QUrl(testServerResource());
     QSignalSpy spyClientFinished(&client, SIGNAL(finished(QCoapReply *)));
@@ -462,6 +480,8 @@ void tst_QCoapClient::multipleRequests()
 void tst_QCoapClient::socketError()
 {
 #ifdef QT_BUILD_INTERNAL
+    CHECK_FOR_COAP_SERVER;
+
     QCoapClientForSocketErrorTests client;
     QUrl url = QUrl(testServerResource());
 
@@ -494,6 +514,8 @@ void tst_QCoapClient::timeout_data()
 void tst_QCoapClient::timeout()
 {
 #ifdef QT_BUILD_INTERNAL
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(uint, timeout);
     QFETCH(uint, maximumRetransmitCount);
 
@@ -542,10 +564,13 @@ void tst_QCoapClient::timeout()
 
 void tst_QCoapClient::abort()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QCoapClient client;
     QUrl url = QUrl(testServerUrl() + "/large");
 
     QScopedPointer<QCoapReply> reply(client.get(url));
+    QVERIFY(!reply.isNull());
     QSignalSpy spyReplyFinished(reply.data(), &QCoapReply::finished);
     QSignalSpy spyReplyAborted(reply.data(), &QCoapReply::aborted);
     QSignalSpy spyReplyError(reply.data(), &QCoapReply::error);
@@ -617,6 +642,8 @@ void tst_QCoapClient::blockwiseReply_data()
 
 void tst_QCoapClient::blockwiseReply()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QUrl, url);
     QFETCH(QCoapMessage::Type, type);
     QFETCH(QByteArray, replyData);
@@ -629,6 +656,7 @@ void tst_QCoapClient::blockwiseReply()
 
     request.setType(type);
     QScopedPointer<QCoapReply> reply(client.get(request));
+    QVERIFY(!reply.isNull());
     QSignalSpy spyReplyFinished(reply.data(), &QCoapReply::finished);
     QSignalSpy spyReplyError(reply.data(), &QCoapReply::error);
     Helper helper;
@@ -666,6 +694,8 @@ void tst_QCoapClient::blockwiseRequest_data()
 
 void tst_QCoapClient::blockwiseRequest()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QUrl, url);
     QFETCH(QCoapMessage::Type, type);
     QFETCH(QByteArray, requestData);
@@ -680,6 +710,7 @@ void tst_QCoapClient::blockwiseRequest()
     request.addOption(QCoapOption::ContentFormat);
 
     QScopedPointer<QCoapReply> reply(client.post(request, requestData));
+    QVERIFY(!reply.isNull());
     QSignalSpy spyReplyFinished(reply.data(), SIGNAL(finished(QCoapReply *)));
 
     QTRY_COMPARE_WITH_TIMEOUT(spyReplyFinished.count(), 1, 30000);
@@ -703,12 +734,15 @@ void tst_QCoapClient::discover_data()
 
 void tst_QCoapClient::discover()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QUrl, url);
     QFETCH(int, resourceNumber);
 
     QCoapClient client;
 
     QScopedPointer<QCoapResourceDiscoveryReply> resourcesReply(client.discover(url)); // /.well-known/core
+    QVERIFY(!resourcesReply.isNull());
     QSignalSpy spyReplyFinished(resourcesReply.data(), SIGNAL(finished(QCoapReply *)));
 
     QTRY_COMPARE_WITH_TIMEOUT(spyReplyFinished.count(), 1, 30000);
@@ -768,6 +802,8 @@ void tst_QCoapClient::observe_data()
 
 void tst_QCoapClient::observe()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QUrl, url);
     QFETCH(QCoapMessage::Type, type);
 
@@ -777,6 +813,7 @@ void tst_QCoapClient::observe()
     request.setType(type);
     QSharedPointer<QCoapReply> reply(client.observe(request),
                                      &QObject::deleteLater);
+    QVERIFY(!reply.isNull());
     QSignalSpy spyReplyNotified(reply.data(), &QCoapReply::notified);
     QSignalSpy spyReplyFinished(reply.data(), &QCoapReply::finished);
 
@@ -907,7 +944,7 @@ void tst_QCoapClient::setMinimumTokenSize()
         QScopedPointer<QCoapReply> reply;
         reply.reset(client.get(QCoapRequest("127.0.0.1")));
 
-        QTRY_COMPARE_WITH_TIMEOUT(spyClientError.count(), 1, 10);
+        QTRY_COMPARE_WITH_TIMEOUT(spyClientError.count(), 1, 100);
         QVERIFY(reply->request().tokenLength() >= expectedMinSize);
         QVERIFY(reply->request().tokenLength() <= maxSize);
     }
diff --git a/tests/auto/qcoapqudpconnection/BLACKLIST b/tests/auto/qcoapqudpconnection/BLACKLIST
new file mode 100644
index 0000000..3719b37
--- /dev/null
+++ b/tests/auto/qcoapqudpconnection/BLACKLIST
@@ -0,0 +1,3 @@
+rhel ci
+opensuse ci
+opensuse-leap ci
diff --git a/tests/auto/qcoapqudpconnection/qcoapqudpconnection.pro b/tests/auto/qcoapqudpconnection/qcoapqudpconnection.pro
index c10d6e6..984d62d 100644
--- a/tests/auto/qcoapqudpconnection/qcoapqudpconnection.pro
+++ b/tests/auto/qcoapqudpconnection/qcoapqudpconnection.pro
@@ -7,3 +7,6 @@ HEADERS += ../coapnetworksettings.h
 
 SOURCES += \
     tst_qcoapqudpconnection.cpp
+
+CONFIG += unsupported/testserver
+QT_TEST_SERVER_LIST = californium
diff --git a/tests/auto/qcoapqudpconnection/tst_qcoapqudpconnection.cpp b/tests/auto/qcoapqudpconnection/tst_qcoapqudpconnection.cpp
index 8876214..fb30392 100644
--- a/tests/auto/qcoapqudpconnection/tst_qcoapqudpconnection.cpp
+++ b/tests/auto/qcoapqudpconnection/tst_qcoapqudpconnection.cpp
@@ -50,6 +50,7 @@ class tst_QCoapQUdpConnection : public QObject
     Q_OBJECT
 
 private Q_SLOTS:
+    void initTestCase();
     void ctor();
     void connectToHost();
     void reconnect();
@@ -72,6 +73,13 @@ public:
     }
 };
 
+void tst_QCoapQUdpConnection::initTestCase()
+{
+#if defined(COAP_TEST_SERVER_IP) || defined(QT_TEST_SERVER)
+    QVERIFY2(waitForHost(testServerHost()), "Failed to connect to Californium plugtest server.");
+#endif
+}
+
 void tst_QCoapQUdpConnection::ctor()
 {
     QCoapQUdpConnection connection;
@@ -166,6 +174,8 @@ void tst_QCoapQUdpConnection::sendRequest_data()
 
 void tst_QCoapQUdpConnection::sendRequest()
 {
+    CHECK_FOR_COAP_SERVER;
+
     QFETCH(QString, protocol);
     QFETCH(QString, host);
     QFETCH(QString, path);
diff --git a/tests/testserver/californium/californium.sh b/tests/testserver/californium/californium.sh
new file mode 100755
index 0000000..82072e4
--- /dev/null
+++ b/tests/testserver/californium/californium.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+#############################################################################
+##
+## Copyright (C) 2019 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the test suite of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 or (at your option) any later version
+## approved by the KDE Free Qt Foundation. The licenses are as published by
+## the Free Software Foundation and appearing in the file LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+java -jar /root/src/californium/demo-apps/run/cf-plugtest-server-1.1.0-SNAPSHOT.jar &
diff --git a/tests/testserver/docker-compose.yml b/tests/testserver/docker-compose.yml
new file mode 100644
index 0000000..44ca4ae
--- /dev/null
+++ b/tests/testserver/docker-compose.yml
@@ -0,0 +1,44 @@
+version: '2.1'
+
+# The tag of images is used by docker compose file to launch the corresponding
+# docker containers. The value of tag comes from the provisioning script
+# (coin/provisioning/.../testserver/docker_testserver.sh). The script gets SHA-1
+# of each server context as the tag of docker images. If one of the server
+# contexts gets changes, please make sure to update this compose file as well.
+# You can run command 'docker images' to list all the tag of test server images.
+# For example:
+# REPOSITORY              TAG
+# qt-test-server-apache2  537fe302f61851d1663f41495230d8e3554a4a13
+
+services:
+    californium:
+        extends:
+            file: ${SHARED_DATA}/docker-compose-common.yml
+            service: ${SHARED_SERVICE}
+        container_name: qt-test-server-californium
+        hostname: ${HOST_NAME:-californium}
+        build:
+            context: .
+            args:
+                provisioningImage: qt-test-server-californium:dd4a2ab0b113fbd45c5d6e92b43341f5eb521926
+                serviceDir: ./californium
+        ports:
+            - "5683:5683/udp"
+            - "5684:5684/udp"
+        entrypoint: ./startup.sh
+        command: service/californium.sh
+    freecoap:
+        extends:
+            file: ${SHARED_DATA}/docker-compose-common.yml
+            service: ${SHARED_SERVICE}
+        container_name: qt-test-server-freecoap
+        hostname: ${HOST_NAME:-freecoap}
+        build:
+            context: .
+            args:
+                provisioningImage: qt-test-server-freecoap:e2d7208ea82e623e2769d9ee1f052b58537d2f35
+                serviceDir: ./freecoap
+        ports:
+            - "5685:5685/udp"
+        entrypoint: ./startup.sh
+        command: service/freecoap.sh
diff --git a/tests/testserver/freecoap/freecoap.sh b/tests/testserver/freecoap/freecoap.sh
new file mode 100755
index 0000000..4b15df6
--- /dev/null
+++ b/tests/testserver/freecoap/freecoap.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+#############################################################################
+##
+## Copyright (C) 2019 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the test suite of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 or (at your option) any later version
+## approved by the KDE Free Qt Foundation. The licenses are as published by
+## the Free Software Foundation and appearing in the file LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+(cd /root/src/FreeCoAP/sample/time_server && ./time_server 0.0.0.0 5685 &)