diff options
author | Dimitrios Apostolou <jimis@qt.io> | 2019-11-05 18:21:19 +0100 |
---|---|---|
committer | Dimitrios Apostolou <jimis@qt.io> | 2020-01-13 19:44:45 +0100 |
commit | fe4a724a3ede5383eab0df67d3e8826070ed2a5a (patch) | |
tree | 9d0df3a79b55bf1dc7bab024b772565dd789ca86 /coin/provisioning/common | |
parent | 302b228c7a726046f28f6899e5dcf60224d58707 (diff) |
Use internal mirror repositories for Ubuntu docker containers
Depending on different external URLs makes our provisioning scripts flaky,
because they can fail if any one of the external servers goes down.
In this patch
* the docker images qt_ubuntu_16.04 and qt_ubuntu_18.04 are being built first,
they apt-get from the internal ubuntu mirror
* all the existing docker images have been rebased to these two images, so
they inherit this behavior
* images that fetch files from the internet have been modified to not do the
fetching themselves; insted URLs are fetched in advance using "DownloadURL"
which accepts internal mirrored URLs, and the downloaded files are ADDed to
the images with a local copy
* previously existing git clone commands have been modified to fetch
internally mirrored tarballs
Task-number: QTQAINFRA-3343
Change-Id: Ic4640a4f7776e38623edba90f0eba7cb68631223
Reviewed-by: Sona Kurazyan <sona.kurazyan@qt.io>
Reviewed-by: Tony Sarajärvi <tony.sarajarvi@qt.io>
Diffstat (limited to 'coin/provisioning/common')
23 files changed, 142 insertions, 72 deletions
diff --git a/coin/provisioning/common/linux/docker.sh b/coin/provisioning/common/linux/docker.sh index 35089e41..cf2ff717 100755 --- a/coin/provisioning/common/linux/docker.sh +++ b/coin/provisioning/common/linux/docker.sh @@ -36,7 +36,9 @@ set -e -. $(dirname "$0")/../../common/unix/DownloadURL.sh +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh localRepo=http://ci-files01-hki.intra.qt.io/input/docker @@ -61,7 +63,7 @@ sudo docker --version # Download and install the docker-compose extension from https://github.com/docker/compose/releases f=docker-compose-$(uname -s)-$(uname -m) DownloadURL \ - $localRepo/$f \ + $localRepo/$f-1.24.1 \ https://github.com/docker/compose/releases/download/1.24.1/$f \ cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose diff --git a/coin/provisioning/common/shared/testserver/apache2/Dockerfile b/coin/provisioning/common/shared/testserver/apache2/Dockerfile index b39eb21e..c3dc5d59 100644 --- a/coin/provisioning/common/shared/testserver/apache2/Dockerfile +++ b/coin/provisioning/common/shared/testserver/apache2/Dockerfile @@ -1,7 +1,8 @@ -FROM ubuntu:16.04 -ARG packages="apache2 libcgi-session-perl wget avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon apache2 libcgi-session-perl" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 80 443 # install configurations and test data -RUN wget https://tools.ietf.org/rfc/rfc3252.txt + +COPY rfc3252.txt . diff --git a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile index dff9fe43..1a629c47 100644 --- a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="apache2 libcgi-session-perl avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon apache2 libcgi-session-perl" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 80 443 diff --git a/coin/provisioning/common/shared/testserver/californium/Dockerfile b/coin/provisioning/common/shared/testserver/californium/Dockerfile index 90722510..041c3ef4 100644 --- a/coin/provisioning/common/shared/testserver/californium/Dockerfile +++ b/coin/provisioning/common/shared/testserver/californium/Dockerfile @@ -1,15 +1,13 @@ -FROM ubuntu:18.04 - -RUN apt-get update && apt-get -y install git maven default-jdk avahi-daemon - -RUN mkdir -p /root/src/ +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon maven default-jdk" +RUN apt-get update && apt-get -y install $packages # Get californium-based CoAP test server WORKDIR /root/src -RUN git clone https://github.com/selart/californium.git +ADD californium-*.tar.gz . +RUN mv californium-* californium WORKDIR /root/src/californium RUN mvn clean install -q -DskipTests - WORKDIR / EXPOSE 5683/udp 5684/udp diff --git a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile index 55486175..4708ef5b 100644 --- a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile +++ b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="cyrus-imapd avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon cyrus-imapd" RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages EXPOSE 143 993 diff --git a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile index 2e525ec6..c20ed4f9 100644 --- a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="cyrus-imapd avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon cyrus-imapd" RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages EXPOSE 143 993 diff --git a/coin/provisioning/common/shared/testserver/danted/Dockerfile b/coin/provisioning/common/shared/testserver/danted/Dockerfile index 67c1159b..0d3ca461 100644 --- a/coin/provisioning/common/shared/testserver/danted/Dockerfile +++ b/coin/provisioning/common/shared/testserver/danted/Dockerfile @@ -1,8 +1,9 @@ -FROM ubuntu:16.04 -ARG packages="gdebi-core wget avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages -RUN wget http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb -RUN gdebi -n dante-server_1.4.1-1_amd64.deb +COPY dante-server_1.4.1-1_amd64.deb . +RUN apt -y install ./dante-server_1.4.1-1_amd64.deb \ + && rm -f ./dante-server_1.4.1-1_amd64.deb EXPOSE 1080-1081 # install configurations and test data diff --git a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile index f48794af..ba732bff 100644 --- a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="dante-server avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon dante-server" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 1080-1081 diff --git a/coin/provisioning/common/shared/testserver/docker_images.sh b/coin/provisioning/common/shared/testserver/docker_images.sh index 1d467f7b..406a5b5d 100755 --- a/coin/provisioning/common/shared/testserver/docker_images.sh +++ b/coin/provisioning/common/shared/testserver/docker_images.sh @@ -33,13 +33,13 @@ ## ############################################################################# -set -ex +set -e + + +PROVISIONING_DIR="$(dirname "$0")/../../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh -[ -x "$(command -v realpath)" ] && FILE=$(realpath ${BASH_SOURCE[0]}) || FILE=${BASH_SOURCE[0]} -case $FILE in - */*) SERVER_PATH="${FILE%/*}" ;; - *) SERVER_PATH="." ;; -esac # Sort files by their SHA-1, and then return the accumulated result sha1tree () { @@ -51,20 +51,76 @@ sha1tree () { sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1 } -# A tag labels a specific image version. In the docker compose file -# (docker-compose.yml) that launches the containers, the tag used is -# "latest". Here the images are also tagged with the SHA1 of each server -# context, so if needed we can modify docker-compose.yml to launch a very -# specific image, thus providing a way to stage backwards-incompatible changes -# across repositories. -source "$SERVER_PATH/settings.sh" +SERVER_PATH="$PROVISIONING_DIR/common/shared/testserver" + +. "$SERVER_PATH/settings.sh" + + +# Download all necessary dependencies outside of the dockerfiles, so that we +# can use provisioning functionality for cached and verified downloads. In the +# dockerfiles we just do COPY to put them where needed. + +echo 'Downloading support files for the docker images' + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/rfc3252.txt \ + https://tools.ietf.org/rfc/rfc3252.txt \ + 50c323dedce95e4fdc2db35cd1b8ebf9d74711bf5296ef438b88d186d7dd082d +cp rfc3252.txt "$SERVER_PATH/vsftpd/" +cp rfc3252.txt "$SERVER_PATH/apache2/" + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/dante-server_1.4.1-1_amd64.deb \ + http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb \ + 674a06f356cebd92c64920cec38a6687650a6f880198fbbad05aaaccca5c0a21 +mv dante-server_1.4.1-1_amd64.deb "$SERVER_PATH/danted/" + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/FreeCoAP-0.7.tar.gz \ + https://github.com/keith-cullen/FreeCoAP/archive/v0.7.tar.gz \ + fa6602e27dc8eaee6e34ff53400c0519da0c5c7cd47bf6f13acb564f52a693ee \ + FreeCoAP-0.7.tar.gz +mv FreeCoAP-0.7.tar.gz "$SERVER_PATH/freecoap/" + +# Custom fork of Eclipse Californium with changes not upstream +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/californium-secure-test-server.tar.gz \ + https://github.com/sonakur/californium/archive/secure-test-server.tar.gz \ + 0ee7f5d4366b9e31f6d2d42e389cb7a66d2db54987b700a38a3a31e8f38a7a19 \ + californium-secure-test-server.tar.gz +mv californium-secure-test-server.tar.gz "$SERVER_PATH/californium/" + + +echo 'Building the docker images...' + +# Build the 2 base layers: qt_ubuntu_1604, qt_ubuntu_1804. +# These are the base for all other docker images. +for image in qt_ubuntu_16.04 qt_ubuntu_18.04 +do + docker build -t $image \ + --build-arg COIN_RUNS_IN_QT_COMPANY="$COIN_RUNS_IN_QT_COMPANY" \ + "$SERVER_PATH/$image" +done + for server in $testserver do + + # We label each docker image with `-t name:tag`. + # A tag labels a specific image version. In the docker compose file + # (docker-compose.yml) that launches the containers, the tag used is + # "latest". Here the images are additionally tagged with the SHA1 of each + # image directory (context), so that if needed we can modify + # docker-compose.yml and modify "latest" to a SHA in order to launch a + # very specific image, thus providing a way to stage + # backwards-incompatible changes across repositories. + context="$SERVER_PATH/$server" tag=$(sha1tree $context) - docker build -t qt-test-server-$server:$tag -t qt-test-server-$server:latest $context + docker build -t qt-test-server-$server:latest \ + -t qt-test-server-$server:$tag \ + $context done docker images diff --git a/coin/provisioning/common/shared/testserver/echo/Dockerfile b/coin/provisioning/common/shared/testserver/echo/Dockerfile index 653ca714..a20419c6 100644 --- a/coin/provisioning/common/shared/testserver/echo/Dockerfile +++ b/coin/provisioning/common/shared/testserver/echo/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="xinetd avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon xinetd" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 7 7/UDP 13 diff --git a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile index 37f488cb..e86357a2 100644 --- a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="xinetd avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon xinetd" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 7 7/UDP 13 diff --git a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile index 2c665bd5..7b2a27a7 100644 --- a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile +++ b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile @@ -1,20 +1,14 @@ -FROM ubuntu:18.04 +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon autoconf automake libtool make libgnutls28-dev" +RUN apt-get update && apt-get -y install $packages -RUN apt-get update && apt-get -y install git avahi-daemon autoconf automake libtool make libgnutls28-dev - -RUN mkdir -p /root/src/ - -# Get FreeCoAP test server WORKDIR /root/src -RUN git clone https://github.com/keith-cullen/FreeCoAP.git +ADD FreeCoAP-*.tar.gz . +RUN mv FreeCoAP-* FreeCoAP WORKDIR /root/src/FreeCoAP -RUN autoreconf --install -RUN ./configure +RUN autoreconf --install && ./configure && make && make install +WORKDIR sample/time_server RUN make -RUN make install -WORKDIR /root/src/FreeCoAP/sample/time_server -RUN make - WORKDIR / EXPOSE 5685/udp diff --git a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile index c719c1eb..de3e5b86 100644 --- a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile +++ b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="ftp-proxy avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon ftp-proxy" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 2121 diff --git a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile index 0e7146ba..383ba140 100644 --- a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="ftp-proxy avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon ftp-proxy" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 2121 diff --git a/coin/provisioning/common/shared/testserver/iptables/Dockerfile b/coin/provisioning/common/shared/testserver/iptables/Dockerfile index cbee61d4..1c2a4b4d 100644 --- a/coin/provisioning/common/shared/testserver/iptables/Dockerfile +++ b/coin/provisioning/common/shared/testserver/iptables/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="iptables avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon iptables" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 1357 diff --git a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile index 5865e9c8..49b277c7 100644 --- a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="iptables avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon iptables" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 1357 diff --git a/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile new file mode 100644 index 00000000..a9c8b9ef --- /dev/null +++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:16.04 +ARG COIN_RUNS_IN_QT_COMPANY +RUN test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue \ + && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \ + || echo "Internal package repository not found. Using public repositories." diff --git a/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile new file mode 100644 index 00000000..f62a29c6 --- /dev/null +++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:18.04 +ARG COIN_RUNS_IN_QT_COMPANY +RUN test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue \ + && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \ + || echo "Internal package repository not found. Using public repositories." diff --git a/coin/provisioning/common/shared/testserver/squid/Dockerfile b/coin/provisioning/common/shared/testserver/squid/Dockerfile index f0ed68b3..330d3929 100644 --- a/coin/provisioning/common/shared/testserver/squid/Dockerfile +++ b/coin/provisioning/common/shared/testserver/squid/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="squid avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon squid" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 3128-3130 diff --git a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile index 208feec7..239ea4c8 100644 --- a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="squid avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon squid" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 3128-3130 diff --git a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile index af6cd7c7..4c2e00e4 100644 --- a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile +++ b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile @@ -1,7 +1,7 @@ -FROM ubuntu:16.04 -ARG packages="vsftpd ftp wget avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon vsftpd ftp" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 20-21 # install configurations and test data -RUN wget https://tools.ietf.org/rfc/rfc3252.txt +COPY rfc3252.txt . diff --git a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile index 843472fd..8886308c 100644 --- a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="vsftpd avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon vsftpd" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 20-21 diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh index aca5dd04..9f74b32e 100644 --- a/coin/provisioning/common/unix/common.sourced.sh +++ b/coin/provisioning/common/unix/common.sourced.sh @@ -43,6 +43,7 @@ export PROVISIONING_ARCH export CMD_PKG_INSTALL export CMD_PKG_LOCALINSTALL export CMD_INSTALL +export COIN_RUNS_IN_QT_COMPANY @@ -143,6 +144,13 @@ set_common_environment () { esac CMD_INSTALL="sudo install" + + COIN_RUNS_IN_QT_COMPANY=false + if ping -c1 repo-clones.ci.qt.io >/dev/null 2>&1 + then + COIN_RUNS_IN_QT_COMPANY=true + fi + } set_common_environment |