Use internal mirror repositories for Ubuntu docker containers

Depending on different external URLs makes our provisioning scripts flaky,
because they can fail if any one of the external servers goes down.
In this patch

* the docker images qt_ubuntu_16.04 and qt_ubuntu_18.04 are being built first,
  they apt-get from the internal ubuntu mirror

* all the existing docker images have been rebased to these two images, so
  they inherit this behavior

* images that fetch files from the internet have been modified to not do the
  fetching themselves; insted URLs are fetched in advance using "DownloadURL"
  which accepts internal mirrored URLs, and the downloaded files are ADDed to
  the images with a local copy

* previously existing git clone commands have been modified to fetch
  internally mirrored tarballs

Task-number: QTQAINFRA-3343
Change-Id: Ic4640a4f7776e38623edba90f0eba7cb68631223
Reviewed-by: Sona Kurazyan <sona.kurazyan@qt.io>
Reviewed-by: Tony Sarajärvi <tony.sarajarvi@qt.io>

23 files changed, 142 insertions, 72 deletions

diff --git a/coin/provisioning/common/linux/docker.sh b/coin/provisioning/common/linux/docker.sh
index 35089e41..cf2ff717 100755
--- a/coin/provisioning/common/linux/docker.sh
+++ b/coin/provisioning/common/linux/docker.sh
@@ -36,7 +36,9 @@
 set -e
 
 
-. $(dirname "$0")/../../common/unix/DownloadURL.sh
+PROVISIONING_DIR="$(dirname "$0")/../../"
+. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
+. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh
 
 
 localRepo=http://ci-files01-hki.intra.qt.io/input/docker
@@ -61,7 +63,7 @@ sudo docker --version
 # Download and install the docker-compose extension from https://github.com/docker/compose/releases
 f=docker-compose-$(uname -s)-$(uname -m)
 DownloadURL  \
-    $localRepo/$f  \
+    $localRepo/$f-1.24.1  \
     https://github.com/docker/compose/releases/download/1.24.1/$f \
     cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a
 sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose
diff --git a/coin/provisioning/common/shared/testserver/apache2/Dockerfile b/coin/provisioning/common/shared/testserver/apache2/Dockerfile
index b39eb21e..c3dc5d59 100644
--- a/coin/provisioning/common/shared/testserver/apache2/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/apache2/Dockerfile
@@ -1,7 +1,8 @@
-FROM ubuntu:16.04
-ARG packages="apache2 libcgi-session-perl wget avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon apache2 libcgi-session-perl"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 80 443
 
 # install configurations and test data
-RUN wget https://tools.ietf.org/rfc/rfc3252.txt
+
+COPY rfc3252.txt .
diff --git a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile
index dff9fe43..1a629c47 100644
--- a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="apache2 libcgi-session-perl avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon apache2 libcgi-session-perl"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 80 443
diff --git a/coin/provisioning/common/shared/testserver/californium/Dockerfile b/coin/provisioning/common/shared/testserver/californium/Dockerfile
index 90722510..041c3ef4 100644
--- a/coin/provisioning/common/shared/testserver/californium/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/californium/Dockerfile
@@ -1,15 +1,13 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get -y install git maven default-jdk avahi-daemon
-
-RUN mkdir -p /root/src/
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon maven default-jdk"
+RUN apt-get update && apt-get -y install $packages
 
 # Get californium-based CoAP test server
 WORKDIR /root/src
-RUN git clone https://github.com/selart/californium.git
+ADD californium-*.tar.gz .
+RUN mv californium-* californium
 WORKDIR /root/src/californium
 RUN mvn clean install -q -DskipTests
-
 WORKDIR /
 
 EXPOSE 5683/udp 5684/udp
diff --git a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile
index 55486175..4708ef5b 100644
--- a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="cyrus-imapd avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon cyrus-imapd"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages
 EXPOSE 143 993
diff --git a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile
index 2e525ec6..c20ed4f9 100644
--- a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="cyrus-imapd avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon cyrus-imapd"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages
 EXPOSE 143 993
diff --git a/coin/provisioning/common/shared/testserver/danted/Dockerfile b/coin/provisioning/common/shared/testserver/danted/Dockerfile
index 67c1159b..0d3ca461 100644
--- a/coin/provisioning/common/shared/testserver/danted/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/danted/Dockerfile
@@ -1,8 +1,9 @@
-FROM ubuntu:16.04
-ARG packages="gdebi-core wget avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
-RUN wget http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb
-RUN gdebi -n dante-server_1.4.1-1_amd64.deb
+COPY dante-server_1.4.1-1_amd64.deb .
+RUN  apt -y install ./dante-server_1.4.1-1_amd64.deb  \
+  && rm -f          ./dante-server_1.4.1-1_amd64.deb
 EXPOSE 1080-1081
 
 # install configurations and test data
diff --git a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile
index f48794af..ba732bff 100644
--- a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="dante-server avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon dante-server"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 1080-1081
diff --git a/coin/provisioning/common/shared/testserver/docker_images.sh b/coin/provisioning/common/shared/testserver/docker_images.sh
index 1d467f7b..406a5b5d 100755
--- a/coin/provisioning/common/shared/testserver/docker_images.sh
+++ b/coin/provisioning/common/shared/testserver/docker_images.sh
@@ -33,13 +33,13 @@
 ##
 #############################################################################
 
-set -ex
+set -e
+
+
+PROVISIONING_DIR="$(dirname "$0")/../../../"
+. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
+. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh
 
-[ -x "$(command -v realpath)" ] && FILE=$(realpath ${BASH_SOURCE[0]}) || FILE=${BASH_SOURCE[0]}
-case $FILE in
-    */*) SERVER_PATH="${FILE%/*}" ;;
-    *) SERVER_PATH="." ;;
-esac
 
 # Sort files by their SHA-1, and then return the accumulated result
 sha1tree () {
@@ -51,20 +51,76 @@ sha1tree () {
         sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1
 }
 
-# A tag labels a specific image version. In the docker compose file
-# (docker-compose.yml) that launches the containers, the tag used is
-# "latest". Here the images are also tagged with the SHA1 of each server
-# context, so if needed we can modify docker-compose.yml to launch a very
-# specific image, thus providing a way to stage backwards-incompatible changes
-# across repositories.
 
-source "$SERVER_PATH/settings.sh"
+SERVER_PATH="$PROVISIONING_DIR/common/shared/testserver"
+
+. "$SERVER_PATH/settings.sh"
+
+
+# Download all necessary dependencies outside of the dockerfiles, so that we
+# can use provisioning functionality for cached and verified downloads. In the
+# dockerfiles we just do COPY to put them where needed.
+
+echo 'Downloading support files for the docker images'
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/rfc3252.txt  \
+    https://tools.ietf.org/rfc/rfc3252.txt  \
+    50c323dedce95e4fdc2db35cd1b8ebf9d74711bf5296ef438b88d186d7dd082d
+cp rfc3252.txt "$SERVER_PATH/vsftpd/"
+cp rfc3252.txt "$SERVER_PATH/apache2/"
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/dante-server_1.4.1-1_amd64.deb  \
+    http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb  \
+    674a06f356cebd92c64920cec38a6687650a6f880198fbbad05aaaccca5c0a21
+mv dante-server_1.4.1-1_amd64.deb "$SERVER_PATH/danted/"
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/FreeCoAP-0.7.tar.gz  \
+    https://github.com/keith-cullen/FreeCoAP/archive/v0.7.tar.gz  \
+    fa6602e27dc8eaee6e34ff53400c0519da0c5c7cd47bf6f13acb564f52a693ee  \
+    FreeCoAP-0.7.tar.gz
+mv FreeCoAP-0.7.tar.gz "$SERVER_PATH/freecoap/"
+
+# Custom fork of Eclipse Californium with changes not upstream
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/californium-secure-test-server.tar.gz  \
+    https://github.com/sonakur/californium/archive/secure-test-server.tar.gz  \
+    0ee7f5d4366b9e31f6d2d42e389cb7a66d2db54987b700a38a3a31e8f38a7a19  \
+    californium-secure-test-server.tar.gz
+mv californium-secure-test-server.tar.gz "$SERVER_PATH/californium/"
+
+
+echo 'Building the docker images...'
+
+# Build the 2 base layers: qt_ubuntu_1604, qt_ubuntu_1804.
+# These are the base for all other docker images.
+for image in  qt_ubuntu_16.04 qt_ubuntu_18.04
+do
+    docker build -t $image  \
+        --build-arg  COIN_RUNS_IN_QT_COMPANY="$COIN_RUNS_IN_QT_COMPANY"  \
+        "$SERVER_PATH/$image"
+done
+
 
 for server in $testserver
 do
+
+    # We label each docker image with `-t name:tag`.
+    # A tag labels a specific image version. In the docker compose file
+    # (docker-compose.yml) that launches the containers, the tag used is
+    # "latest". Here the images are additionally tagged with the SHA1 of each
+    # image directory (context), so that if needed we can modify
+    # docker-compose.yml and modify "latest" to a SHA in order to launch a
+    # very specific image, thus providing a way to stage
+    # backwards-incompatible changes across repositories.
+
     context="$SERVER_PATH/$server"
     tag=$(sha1tree $context)
-    docker build -t qt-test-server-$server:$tag -t qt-test-server-$server:latest  $context
+    docker build -t qt-test-server-$server:latest  \
+                 -t qt-test-server-$server:$tag    \
+        $context
 done
 
 docker images
diff --git a/coin/provisioning/common/shared/testserver/echo/Dockerfile b/coin/provisioning/common/shared/testserver/echo/Dockerfile
index 653ca714..a20419c6 100644
--- a/coin/provisioning/common/shared/testserver/echo/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/echo/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="xinetd avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon xinetd"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 7 7/UDP 13
diff --git a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile
index 37f488cb..e86357a2 100644
--- a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="xinetd avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon xinetd"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 7 7/UDP 13
diff --git a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile
index 2c665bd5..7b2a27a7 100644
--- a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile
@@ -1,20 +1,14 @@
-FROM ubuntu:18.04
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon autoconf automake libtool make libgnutls28-dev"
+RUN apt-get update && apt-get -y install $packages
 
-RUN apt-get update && apt-get -y install git avahi-daemon autoconf automake libtool make libgnutls28-dev
-
-RUN mkdir -p /root/src/
-
-# Get FreeCoAP test server
 WORKDIR /root/src
-RUN git clone https://github.com/keith-cullen/FreeCoAP.git
+ADD FreeCoAP-*.tar.gz .
+RUN mv FreeCoAP-* FreeCoAP
 WORKDIR /root/src/FreeCoAP
-RUN autoreconf --install
-RUN ./configure
+RUN autoreconf --install && ./configure && make && make install
+WORKDIR sample/time_server
 RUN make
-RUN make install
-WORKDIR /root/src/FreeCoAP/sample/time_server
-RUN make
-
 WORKDIR /
 
 EXPOSE 5685/udp
diff --git a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile
index c719c1eb..de3e5b86 100644
--- a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="ftp-proxy avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon ftp-proxy"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 2121
diff --git a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile
index 0e7146ba..383ba140 100644
--- a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="ftp-proxy avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon ftp-proxy"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 2121
diff --git a/coin/provisioning/common/shared/testserver/iptables/Dockerfile b/coin/provisioning/common/shared/testserver/iptables/Dockerfile
index cbee61d4..1c2a4b4d 100644
--- a/coin/provisioning/common/shared/testserver/iptables/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/iptables/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="iptables avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon iptables"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 1357
diff --git a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile
index 5865e9c8..49b277c7 100644
--- a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="iptables avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon iptables"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 1357
diff --git a/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile
new file mode 100644
index 00000000..a9c8b9ef
--- /dev/null
+++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile
@@ -0,0 +1,5 @@
+FROM ubuntu:16.04
+ARG COIN_RUNS_IN_QT_COMPANY
+RUN  test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue  \
+  && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \
+  || echo "Internal package repository not found. Using public repositories."
diff --git a/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile
new file mode 100644
index 00000000..f62a29c6
--- /dev/null
+++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile
@@ -0,0 +1,5 @@
+FROM ubuntu:18.04
+ARG COIN_RUNS_IN_QT_COMPANY
+RUN  test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue  \
+  && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \
+  || echo "Internal package repository not found. Using public repositories."
diff --git a/coin/provisioning/common/shared/testserver/squid/Dockerfile b/coin/provisioning/common/shared/testserver/squid/Dockerfile
index f0ed68b3..330d3929 100644
--- a/coin/provisioning/common/shared/testserver/squid/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/squid/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="squid avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon squid"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 3128-3130
diff --git a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile
index 208feec7..239ea4c8 100644
--- a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="squid avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon squid"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 3128-3130
diff --git a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile
index af6cd7c7..4c2e00e4 100644
--- a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile
@@ -1,7 +1,7 @@
-FROM ubuntu:16.04
-ARG packages="vsftpd ftp wget avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon vsftpd ftp"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 20-21
 
 # install configurations and test data
-RUN wget https://tools.ietf.org/rfc/rfc3252.txt
+COPY rfc3252.txt .
diff --git a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile
index 843472fd..8886308c 100644
--- a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="vsftpd avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon vsftpd"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 20-21
diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh
index aca5dd04..9f74b32e 100644
--- a/coin/provisioning/common/unix/common.sourced.sh
+++ b/coin/provisioning/common/unix/common.sourced.sh
@@ -43,6 +43,7 @@ export PROVISIONING_ARCH
 export CMD_PKG_INSTALL
 export CMD_PKG_LOCALINSTALL
 export CMD_INSTALL
+export COIN_RUNS_IN_QT_COMPANY
 
 
 
@@ -143,6 +144,13 @@ set_common_environment () {
     esac
 
     CMD_INSTALL="sudo install"
+
+    COIN_RUNS_IN_QT_COMPANY=false
+    if  ping -c1 repo-clones.ci.qt.io  >/dev/null 2>&1
+    then
+        COIN_RUNS_IN_QT_COMPANY=true
+    fi
+
 }
 
 set_common_environment