Support multi-stage builds to provision the configurations and test data

In order to reuse the test server to the external modules, it is much
easier to share the common configurations (scripts) and test data via
Dockerfile. In addition, the external module can create more layers
depending on their needs. Therefore, supporting multi-stage builds is
needed. The disadvantage is that the docker-compose needs to re-build
the images every time. However, it is just a one-time effort. If the
Dockerfile doesn't get changed, the extra build time can be ignored.

Because of multi-stage builds, the test server will keep a Dockerfile at
least. Therefore, the volume sharing is no more needed. The test data of
a service can be added into the images by using COPY/ADD commands.

NOTE:
This patch relies on docker-compose v1.21.0 (docker-compose build now
supports the use of Dockerfile from outside the build context).

Change-Id: Ib3f6a5fcf6979732ae8a40a494a1360fca4ac7bf
Reviewed-by: Maurice Kalinowski <maurice.kalinowski@qt.io>
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

5 files changed, 154 insertions, 125 deletions

diff --git a/tests/testserver/Dockerfile b/tests/testserver/Dockerfile
deleted file mode 100644
index a4db873a5d..0000000000
--- a/tests/testserver/Dockerfile
+++ /dev/null
@@ -1,11 +0,0 @@
-# This Dockerfile is used on windows as volume sharing / mounting does not work in conjunction
-# docker-machine. Windows Update 1809 might solve some of those issues when using docker0
-# network switch, but nothing has been reported in regards to the combination with docker-machine.
-
-ARG provisioningImage
-FROM $provisioningImage
-
-# Common is used for all test images so far, no need for a variable
-COPY ./common /common
-ARG servicedir
-COPY $servicedir /service
diff --git a/tests/testserver/common/startup.sh b/tests/testserver/common/startup.sh
index 74990a47f6..1386314e38 100755
--- a/tests/testserver/common/startup.sh
+++ b/tests/testserver/common/startup.sh
@@ -34,7 +34,7 @@ set -ex
 # export variables
 export USER=qt-test-server
 export PASS=password
-export CONFIG=common/testdata
+export CONFIG=service/testdata
 export TESTDATA=service/testdata
 
 # add users
@@ -43,17 +43,4 @@ useradd -m -s /bin/bash $USER; echo "$USER:$PASS" | chpasswd
 # install configurations and test data
 su $USER -c "cp $CONFIG/system/passwords ~/"
 
-# modules initialization (apache2.sh, ftp-proxy.sh ...)
-for RUN_CMD
-do $RUN_CMD
-done
-
-# start multicast DNS service discovery (mDNS)
-sed -i -e "s,#domain-name=local,domain-name=${test_domain:-test-net.qt.local}," \
-    -e "s,#publish-aaaa-on-ipv4=yes,publish-aaaa-on-ipv4=no," \
-    /etc/avahi/avahi-daemon.conf
-service dbus restart
-service avahi-daemon restart
-
-# keep-alive in docker detach mode
-sleep infinity
+./startup.sh "$@"
diff --git a/tests/testserver/docker-compose-for-macOS.yml b/tests/testserver/docker-compose-for-macOS.yml
index aa610dfb88..c5348e27b6 100644
--- a/tests/testserver/docker-compose-for-macOS.yml
+++ b/tests/testserver/docker-compose-for-macOS.yml
@@ -12,82 +12,95 @@ version: '3.4'
 
 services:
     apache2:
-        image: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
         container_name: qt-test-server-apache2
         domainname: ${TEST_DOMAIN}
-        volumes:
-            - ./common:/common:ro
-            - ./apache2:/service:ro
-        entrypoint: common/startup.sh
-        command: [common/ssl.sh, service/apache2.sh]
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
+                shareDir: ./common
+                serviceDir: ./apache2
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
+        command: [service/ssl.sh, service/apache2.sh]
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
         environment:
-            - test_domain=${TEST_DOMAIN}
             - test_cert="qt-test-server-host-network-cacert.pem"
 
     squid:
-        image: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
         container_name: qt-test-server-squid
         domainname: ${TEST_DOMAIN}
         depends_on:
             - apache2
-        volumes:
-            - ./common:/common:ro
-            - ./squid:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
+                shareDir: ./common
+                serviceDir: ./squid
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/squid.sh
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
-        environment:
-            - test_domain=${TEST_DOMAIN}
 
     vsftpd:
-        image: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
         container_name: qt-test-server-vsftpd
         domainname: ${TEST_DOMAIN}
-        volumes:
-            - ./common:/common:ro
-            - ./vsftpd:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
+                shareDir: ./common
+                serviceDir: ./vsftpd
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/vsftpd.sh
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
-        environment:
-            - test_domain=${TEST_DOMAIN}
 
     ftp-proxy:
-        image: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
         container_name: qt-test-server-ftp-proxy
         domainname: ${TEST_DOMAIN}
         depends_on:
             - vsftpd
-        volumes:
-            - ./common:/common:ro
-            - ./ftp-proxy:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
+                shareDir: ./common
+                serviceDir: ./ftp-proxy
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/ftp-proxy.sh
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
-        environment:
-            - test_domain=${TEST_DOMAIN}
 
     danted:
-        image: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
         container_name: qt-test-server-danted
         domainname: ${TEST_DOMAIN}
         depends_on:
             - apache2
             - vsftpd
             - ftp-proxy
-        volumes:
-            - ./common:/common:ro
-            - ./danted:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
+                shareDir: ./common
+                serviceDir: ./danted
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/danted.sh
         network_mode: "host"
         extra_hosts:
@@ -97,4 +110,4 @@ services:
             - danted_external=${MACHINE_IP}
             - danted_auth_internal=${MACHINE_IP}
             - danted_auth_external=${MACHINE_IP}
-            - test_domain=${TEST_DOMAIN}
+
diff --git a/tests/testserver/docker-compose-for-windows.yml b/tests/testserver/docker-compose-for-windows.yml
index 4867413361..c5348e27b6 100644
--- a/tests/testserver/docker-compose-for-windows.yml
+++ b/tests/testserver/docker-compose-for-windows.yml
@@ -16,16 +16,18 @@ services:
         domainname: ${TEST_DOMAIN}
         build:
             context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
             args:
                 provisioningImage: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
-                servicedir: ./apache2
-        entrypoint: common/startup.sh
-        command: [common/ssl.sh, service/apache2.sh]
+                shareDir: ./common
+                serviceDir: ./apache2
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
+        command: [service/ssl.sh, service/apache2.sh]
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
         environment:
-            - test_domain=${TEST_DOMAIN}
             - test_cert="qt-test-server-host-network-cacert.pem"
 
     squid:
@@ -35,32 +37,34 @@ services:
             - apache2
         build:
             context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
             args:
                 provisioningImage: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
-                servicedir: ./squid
-        entrypoint: common/startup.sh
+                shareDir: ./common
+                serviceDir: ./squid
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/squid.sh
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
-        environment:
-            - test_domain=${TEST_DOMAIN}
 
     vsftpd:
         container_name: qt-test-server-vsftpd
         domainname: ${TEST_DOMAIN}
         build:
             context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
             args:
                 provisioningImage: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
-                servicedir: ./vsftpd
-        entrypoint: common/startup.sh
+                shareDir: ./common
+                serviceDir: ./vsftpd
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/vsftpd.sh
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
-        environment:
-            - test_domain=${TEST_DOMAIN}
 
     ftp-proxy:
         container_name: qt-test-server-ftp-proxy
@@ -69,16 +73,17 @@ services:
             - vsftpd
         build:
             context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
             args:
                 provisioningImage: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
-                servicedir: ./ftp-proxy
-        entrypoint: common/startup.sh
+                shareDir: ./common
+                serviceDir: ./ftp-proxy
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/ftp-proxy.sh
         network_mode: "host"
         extra_hosts:
             - "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
-        environment:
-            - test_domain=${TEST_DOMAIN}
 
     danted:
         container_name: qt-test-server-danted
@@ -89,10 +94,13 @@ services:
             - ftp-proxy
         build:
             context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
             args:
                 provisioningImage: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
-                servicedir: ./danted
-        entrypoint: common/startup.sh
+                shareDir: ./common
+                serviceDir: ./danted
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/danted.sh
         network_mode: "host"
         extra_hosts:
@@ -102,4 +110,4 @@ services:
             - danted_external=${MACHINE_IP}
             - danted_auth_internal=${MACHINE_IP}
             - danted_auth_external=${MACHINE_IP}
-            - test_domain=${TEST_DOMAIN}
+
diff --git a/tests/testserver/docker-compose.yml b/tests/testserver/docker-compose.yml
index 962daad3c9..75e8a0fea2 100644
--- a/tests/testserver/docker-compose.yml
+++ b/tests/testserver/docker-compose.yml
@@ -12,18 +12,21 @@ version: '3.4'
 
 services:
     apache2:
-        image: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
         container_name: qt-test-server-apache2
         domainname: ${TEST_DOMAIN}
         hostname: apache2
-        volumes:
-            - ./common:/common:ro
-            - ./apache2:/service:ro
-        entrypoint: common/startup.sh
-        command: [common/ssl.sh, service/apache2.sh]
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
+                shareDir: ./common
+                serviceDir: ./apache2
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
+        command: [service/ssl.sh, service/apache2.sh]
 
     squid:
-        image: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
         container_name: qt-test-server-squid
         domainname: ${TEST_DOMAIN}
         hostname: squid
@@ -35,25 +38,33 @@ services:
             - iptables:iptables.${TEST_DOMAIN}
             - vsftpd:vsftpd.${TEST_DOMAIN}
             - echo:echo.${TEST_DOMAIN}
-        volumes:
-            - ./common:/common:ro
-            - ./squid:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
+                shareDir: ./common
+                serviceDir: ./squid
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/squid.sh
 
     vsftpd:
-        image: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
         container_name: qt-test-server-vsftpd
         domainname: ${TEST_DOMAIN}
         hostname: vsftpd
-        volumes:
-            - ./common:/common:ro
-            - ./vsftpd:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
+                shareDir: ./common
+                serviceDir: ./vsftpd
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/vsftpd.sh
 
     ftp-proxy:
-        image: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
         container_name: qt-test-server-ftp-proxy
         domainname: ${TEST_DOMAIN}
         hostname: ftp-proxy
@@ -61,14 +72,18 @@ services:
             - vsftpd
         external_links:
             - vsftpd:vsftpd.${TEST_DOMAIN}
-        volumes:
-            - ./common:/common:ro
-            - ./ftp-proxy:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
+                shareDir: ./common
+                serviceDir: ./ftp-proxy
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/ftp-proxy.sh
 
     danted:
-        image: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
         container_name: qt-test-server-danted
         domainname: ${TEST_DOMAIN}
         hostname: danted
@@ -82,44 +97,61 @@ services:
             - ftp-proxy:ftp-proxy.${TEST_DOMAIN}
             - cyrus:cyrus.${TEST_DOMAIN}
             - echo:echo.${TEST_DOMAIN}
-        volumes:
-            - ./common:/common:ro
-            - ./danted:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
+                shareDir: ./common
+                serviceDir: ./danted
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/danted.sh
 
     cyrus:
-        image: qt-test-server-cyrus:c8d72754abc0e501afd624ce838e4df35505abc9
         container_name: qt-test-server-cyrus
         domainname: ${TEST_DOMAIN}
         hostname: cyrus
-        volumes:
-            - ./common:/common:ro
-            - ./cyrus:/service:ro
-        entrypoint: common/startup.sh
-        command: [common/ssl.sh, service/cyrus.sh]
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-cyrus:c8d72754abc0e501afd624ce838e4df35505abc9
+                shareDir: ./common
+                serviceDir: ./cyrus
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
+        command: [service/ssl.sh, service/cyrus.sh]
 
     iptables:
-        image: qt-test-server-iptables:cb7a8bd6d28602085a88c8ced7d67e28e75781e2
         container_name: qt-test-server-iptables
         domainname: ${TEST_DOMAIN}
         hostname: iptables
-        volumes:
-            - ./common:/common:ro
-            - ./iptables:/service:ro
-        entrypoint: common/startup.sh
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-iptables:cb7a8bd6d28602085a88c8ced7d67e28e75781e2
+                shareDir: ./common
+                serviceDir: ./iptables
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
         command: service/iptables.sh
         cap_add:
             - NET_ADMIN
             - NET_RAW
 
     echo:
-      image: qt-test-server-echo:b29ad409e746a834c1055fd0f7a55fd5056da6ea
-      container_name: qt-test-server-echo
-      domainname: ${TEST_DOMAIN}
-      hostname: echo
-      volumes:
-          - ./common:/common:ro
-          - ./echo:/service:ro
-      entrypoint: common/startup.sh
-      command: service/echo.sh
+        container_name: qt-test-server-echo
+        domainname: ${TEST_DOMAIN}
+        hostname: echo
+        build:
+            context: .
+            dockerfile: ${SHARED_DATA}/Dockerfile
+            args:
+                provisioningImage: qt-test-server-echo:b29ad409e746a834c1055fd0f7a55fd5056da6ea
+                shareDir: ./common
+                serviceDir: ./echo
+                test_domain: ${TEST_DOMAIN}
+        entrypoint: service/startup.sh
+        command: service/echo.sh