Use internal mirror repositories for Ubuntu docker containers

Depending on different external URLs makes our provisioning scripts flaky,
because they can fail if any one of the external servers goes down.
In this patch

* the docker images qt_ubuntu_16.04 and qt_ubuntu_18.04 are being built first,
  they apt-get from the internal ubuntu mirror

* all the existing docker images have been rebased to these two images, so
  they inherit this behavior

* images that fetch files from the internet have been modified to not do the
  fetching themselves; insted URLs are fetched in advance using "DownloadURL"
  which accepts internal mirrored URLs, and the downloaded files are ADDed to
  the images with a local copy

* previously existing git clone commands have been modified to fetch
  internally mirrored tarballs

Task-number: QTQAINFRA-3343
Change-Id: Ic4640a4f7776e38623edba90f0eba7cb68631223
Reviewed-by: Sona Kurazyan <sona.kurazyan@qt.io>
Reviewed-by: Tony Sarajärvi <tony.sarajarvi@qt.io>

1 files changed, 70 insertions, 14 deletions

diff --git a/coin/provisioning/common/shared/testserver/docker_images.sh b/coin/provisioning/common/shared/testserver/docker_images.sh
index 1d467f7b..406a5b5d 100755
--- a/coin/provisioning/common/shared/testserver/docker_images.sh
+++ b/coin/provisioning/common/shared/testserver/docker_images.sh
@@ -33,13 +33,13 @@
 ##
 #############################################################################
 
-set -ex
+set -e
+
+
+PROVISIONING_DIR="$(dirname "$0")/../../../"
+. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
+. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh
 
-[ -x "$(command -v realpath)" ] && FILE=$(realpath ${BASH_SOURCE[0]}) || FILE=${BASH_SOURCE[0]}
-case $FILE in
-    */*) SERVER_PATH="${FILE%/*}" ;;
-    *) SERVER_PATH="." ;;
-esac
 
 # Sort files by their SHA-1, and then return the accumulated result
 sha1tree () {
@@ -51,20 +51,76 @@ sha1tree () {
         sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1
 }
 
-# A tag labels a specific image version. In the docker compose file
-# (docker-compose.yml) that launches the containers, the tag used is
-# "latest". Here the images are also tagged with the SHA1 of each server
-# context, so if needed we can modify docker-compose.yml to launch a very
-# specific image, thus providing a way to stage backwards-incompatible changes
-# across repositories.
 
-source "$SERVER_PATH/settings.sh"
+SERVER_PATH="$PROVISIONING_DIR/common/shared/testserver"
+
+. "$SERVER_PATH/settings.sh"
+
+
+# Download all necessary dependencies outside of the dockerfiles, so that we
+# can use provisioning functionality for cached and verified downloads. In the
+# dockerfiles we just do COPY to put them where needed.
+
+echo 'Downloading support files for the docker images'
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/rfc3252.txt  \
+    https://tools.ietf.org/rfc/rfc3252.txt  \
+    50c323dedce95e4fdc2db35cd1b8ebf9d74711bf5296ef438b88d186d7dd082d
+cp rfc3252.txt "$SERVER_PATH/vsftpd/"
+cp rfc3252.txt "$SERVER_PATH/apache2/"
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/dante-server_1.4.1-1_amd64.deb  \
+    http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb  \
+    674a06f356cebd92c64920cec38a6687650a6f880198fbbad05aaaccca5c0a21
+mv dante-server_1.4.1-1_amd64.deb "$SERVER_PATH/danted/"
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/FreeCoAP-0.7.tar.gz  \
+    https://github.com/keith-cullen/FreeCoAP/archive/v0.7.tar.gz  \
+    fa6602e27dc8eaee6e34ff53400c0519da0c5c7cd47bf6f13acb564f52a693ee  \
+    FreeCoAP-0.7.tar.gz
+mv FreeCoAP-0.7.tar.gz "$SERVER_PATH/freecoap/"
+
+# Custom fork of Eclipse Californium with changes not upstream
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/californium-secure-test-server.tar.gz  \
+    https://github.com/sonakur/californium/archive/secure-test-server.tar.gz  \
+    0ee7f5d4366b9e31f6d2d42e389cb7a66d2db54987b700a38a3a31e8f38a7a19  \
+    californium-secure-test-server.tar.gz
+mv californium-secure-test-server.tar.gz "$SERVER_PATH/californium/"
+
+
+echo 'Building the docker images...'
+
+# Build the 2 base layers: qt_ubuntu_1604, qt_ubuntu_1804.
+# These are the base for all other docker images.
+for image in  qt_ubuntu_16.04 qt_ubuntu_18.04
+do
+    docker build -t $image  \
+        --build-arg  COIN_RUNS_IN_QT_COMPANY="$COIN_RUNS_IN_QT_COMPANY"  \
+        "$SERVER_PATH/$image"
+done
+
 
 for server in $testserver
 do
+
+    # We label each docker image with `-t name:tag`.
+    # A tag labels a specific image version. In the docker compose file
+    # (docker-compose.yml) that launches the containers, the tag used is
+    # "latest". Here the images are additionally tagged with the SHA1 of each
+    # image directory (context), so that if needed we can modify
+    # docker-compose.yml and modify "latest" to a SHA in order to launch a
+    # very specific image, thus providing a way to stage
+    # backwards-incompatible changes across repositories.
+
     context="$SERVER_PATH/$server"
     tag=$(sha1tree $context)
-    docker build -t qt-test-server-$server:$tag -t qt-test-server-$server:latest  $context
+    docker build -t qt-test-server-$server:latest  \
+                 -t qt-test-server-$server:$tag    \
+        $context
 done
 
 docker images