diff options
23 files changed, 142 insertions, 72 deletions
diff --git a/coin/provisioning/common/linux/docker.sh b/coin/provisioning/common/linux/docker.sh index 35089e41..cf2ff717 100755 --- a/coin/provisioning/common/linux/docker.sh +++ b/coin/provisioning/common/linux/docker.sh @@ -36,7 +36,9 @@ set -e -. $(dirname "$0")/../../common/unix/DownloadURL.sh +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh localRepo=http://ci-files01-hki.intra.qt.io/input/docker @@ -61,7 +63,7 @@ sudo docker --version # Download and install the docker-compose extension from https://github.com/docker/compose/releases f=docker-compose-$(uname -s)-$(uname -m) DownloadURL \ - $localRepo/$f \ + $localRepo/$f-1.24.1 \ https://github.com/docker/compose/releases/download/1.24.1/$f \ cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose diff --git a/coin/provisioning/common/shared/testserver/apache2/Dockerfile b/coin/provisioning/common/shared/testserver/apache2/Dockerfile index b39eb21e..c3dc5d59 100644 --- a/coin/provisioning/common/shared/testserver/apache2/Dockerfile +++ b/coin/provisioning/common/shared/testserver/apache2/Dockerfile @@ -1,7 +1,8 @@ -FROM ubuntu:16.04 -ARG packages="apache2 libcgi-session-perl wget avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon apache2 libcgi-session-perl" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 80 443 # install configurations and test data -RUN wget https://tools.ietf.org/rfc/rfc3252.txt + +COPY rfc3252.txt . diff --git a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile index dff9fe43..1a629c47 100644 --- a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="apache2 libcgi-session-perl avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon apache2 libcgi-session-perl" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 80 443 diff --git a/coin/provisioning/common/shared/testserver/californium/Dockerfile b/coin/provisioning/common/shared/testserver/californium/Dockerfile index 90722510..041c3ef4 100644 --- a/coin/provisioning/common/shared/testserver/californium/Dockerfile +++ b/coin/provisioning/common/shared/testserver/californium/Dockerfile @@ -1,15 +1,13 @@ -FROM ubuntu:18.04 - -RUN apt-get update && apt-get -y install git maven default-jdk avahi-daemon - -RUN mkdir -p /root/src/ +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon maven default-jdk" +RUN apt-get update && apt-get -y install $packages # Get californium-based CoAP test server WORKDIR /root/src -RUN git clone https://github.com/selart/californium.git +ADD californium-*.tar.gz . +RUN mv californium-* californium WORKDIR /root/src/californium RUN mvn clean install -q -DskipTests - WORKDIR / EXPOSE 5683/udp 5684/udp diff --git a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile index 55486175..4708ef5b 100644 --- a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile +++ b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="cyrus-imapd avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon cyrus-imapd" RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages EXPOSE 143 993 diff --git a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile index 2e525ec6..c20ed4f9 100644 --- a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="cyrus-imapd avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon cyrus-imapd" RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages EXPOSE 143 993 diff --git a/coin/provisioning/common/shared/testserver/danted/Dockerfile b/coin/provisioning/common/shared/testserver/danted/Dockerfile index 67c1159b..0d3ca461 100644 --- a/coin/provisioning/common/shared/testserver/danted/Dockerfile +++ b/coin/provisioning/common/shared/testserver/danted/Dockerfile @@ -1,8 +1,9 @@ -FROM ubuntu:16.04 -ARG packages="gdebi-core wget avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages -RUN wget http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb -RUN gdebi -n dante-server_1.4.1-1_amd64.deb +COPY dante-server_1.4.1-1_amd64.deb . +RUN apt -y install ./dante-server_1.4.1-1_amd64.deb \ + && rm -f ./dante-server_1.4.1-1_amd64.deb EXPOSE 1080-1081 # install configurations and test data diff --git a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile index f48794af..ba732bff 100644 --- a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="dante-server avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon dante-server" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 1080-1081 diff --git a/coin/provisioning/common/shared/testserver/docker_images.sh b/coin/provisioning/common/shared/testserver/docker_images.sh index 1d467f7b..406a5b5d 100755 --- a/coin/provisioning/common/shared/testserver/docker_images.sh +++ b/coin/provisioning/common/shared/testserver/docker_images.sh @@ -33,13 +33,13 @@ ## ############################################################################# -set -ex +set -e + + +PROVISIONING_DIR="$(dirname "$0")/../../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh -[ -x "$(command -v realpath)" ] && FILE=$(realpath ${BASH_SOURCE[0]}) || FILE=${BASH_SOURCE[0]} -case $FILE in - */*) SERVER_PATH="${FILE%/*}" ;; - *) SERVER_PATH="." ;; -esac # Sort files by their SHA-1, and then return the accumulated result sha1tree () { @@ -51,20 +51,76 @@ sha1tree () { sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1 } -# A tag labels a specific image version. In the docker compose file -# (docker-compose.yml) that launches the containers, the tag used is -# "latest". Here the images are also tagged with the SHA1 of each server -# context, so if needed we can modify docker-compose.yml to launch a very -# specific image, thus providing a way to stage backwards-incompatible changes -# across repositories. -source "$SERVER_PATH/settings.sh" +SERVER_PATH="$PROVISIONING_DIR/common/shared/testserver" + +. "$SERVER_PATH/settings.sh" + + +# Download all necessary dependencies outside of the dockerfiles, so that we +# can use provisioning functionality for cached and verified downloads. In the +# dockerfiles we just do COPY to put them where needed. + +echo 'Downloading support files for the docker images' + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/rfc3252.txt \ + https://tools.ietf.org/rfc/rfc3252.txt \ + 50c323dedce95e4fdc2db35cd1b8ebf9d74711bf5296ef438b88d186d7dd082d +cp rfc3252.txt "$SERVER_PATH/vsftpd/" +cp rfc3252.txt "$SERVER_PATH/apache2/" + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/dante-server_1.4.1-1_amd64.deb \ + http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb \ + 674a06f356cebd92c64920cec38a6687650a6f880198fbbad05aaaccca5c0a21 +mv dante-server_1.4.1-1_amd64.deb "$SERVER_PATH/danted/" + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/FreeCoAP-0.7.tar.gz \ + https://github.com/keith-cullen/FreeCoAP/archive/v0.7.tar.gz \ + fa6602e27dc8eaee6e34ff53400c0519da0c5c7cd47bf6f13acb564f52a693ee \ + FreeCoAP-0.7.tar.gz +mv FreeCoAP-0.7.tar.gz "$SERVER_PATH/freecoap/" + +# Custom fork of Eclipse Californium with changes not upstream +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/docker/californium-secure-test-server.tar.gz \ + https://github.com/sonakur/californium/archive/secure-test-server.tar.gz \ + 0ee7f5d4366b9e31f6d2d42e389cb7a66d2db54987b700a38a3a31e8f38a7a19 \ + californium-secure-test-server.tar.gz +mv californium-secure-test-server.tar.gz "$SERVER_PATH/californium/" + + +echo 'Building the docker images...' + +# Build the 2 base layers: qt_ubuntu_1604, qt_ubuntu_1804. +# These are the base for all other docker images. +for image in qt_ubuntu_16.04 qt_ubuntu_18.04 +do + docker build -t $image \ + --build-arg COIN_RUNS_IN_QT_COMPANY="$COIN_RUNS_IN_QT_COMPANY" \ + "$SERVER_PATH/$image" +done + for server in $testserver do + + # We label each docker image with `-t name:tag`. + # A tag labels a specific image version. In the docker compose file + # (docker-compose.yml) that launches the containers, the tag used is + # "latest". Here the images are additionally tagged with the SHA1 of each + # image directory (context), so that if needed we can modify + # docker-compose.yml and modify "latest" to a SHA in order to launch a + # very specific image, thus providing a way to stage + # backwards-incompatible changes across repositories. + context="$SERVER_PATH/$server" tag=$(sha1tree $context) - docker build -t qt-test-server-$server:$tag -t qt-test-server-$server:latest $context + docker build -t qt-test-server-$server:latest \ + -t qt-test-server-$server:$tag \ + $context done docker images diff --git a/coin/provisioning/common/shared/testserver/echo/Dockerfile b/coin/provisioning/common/shared/testserver/echo/Dockerfile index 653ca714..a20419c6 100644 --- a/coin/provisioning/common/shared/testserver/echo/Dockerfile +++ b/coin/provisioning/common/shared/testserver/echo/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="xinetd avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon xinetd" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 7 7/UDP 13 diff --git a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile index 37f488cb..e86357a2 100644 --- a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="xinetd avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon xinetd" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 7 7/UDP 13 diff --git a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile index 2c665bd5..7b2a27a7 100644 --- a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile +++ b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile @@ -1,20 +1,14 @@ -FROM ubuntu:18.04 +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon autoconf automake libtool make libgnutls28-dev" +RUN apt-get update && apt-get -y install $packages -RUN apt-get update && apt-get -y install git avahi-daemon autoconf automake libtool make libgnutls28-dev - -RUN mkdir -p /root/src/ - -# Get FreeCoAP test server WORKDIR /root/src -RUN git clone https://github.com/keith-cullen/FreeCoAP.git +ADD FreeCoAP-*.tar.gz . +RUN mv FreeCoAP-* FreeCoAP WORKDIR /root/src/FreeCoAP -RUN autoreconf --install -RUN ./configure +RUN autoreconf --install && ./configure && make && make install +WORKDIR sample/time_server RUN make -RUN make install -WORKDIR /root/src/FreeCoAP/sample/time_server -RUN make - WORKDIR / EXPOSE 5685/udp diff --git a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile index c719c1eb..de3e5b86 100644 --- a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile +++ b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="ftp-proxy avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon ftp-proxy" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 2121 diff --git a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile index 0e7146ba..383ba140 100644 --- a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="ftp-proxy avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon ftp-proxy" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 2121 diff --git a/coin/provisioning/common/shared/testserver/iptables/Dockerfile b/coin/provisioning/common/shared/testserver/iptables/Dockerfile index cbee61d4..1c2a4b4d 100644 --- a/coin/provisioning/common/shared/testserver/iptables/Dockerfile +++ b/coin/provisioning/common/shared/testserver/iptables/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="iptables avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon iptables" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 1357 diff --git a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile index 5865e9c8..49b277c7 100644 --- a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="iptables avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon iptables" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 1357 diff --git a/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile new file mode 100644 index 00000000..a9c8b9ef --- /dev/null +++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:16.04 +ARG COIN_RUNS_IN_QT_COMPANY +RUN test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue \ + && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \ + || echo "Internal package repository not found. Using public repositories." diff --git a/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile new file mode 100644 index 00000000..f62a29c6 --- /dev/null +++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:18.04 +ARG COIN_RUNS_IN_QT_COMPANY +RUN test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue \ + && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \ + || echo "Internal package repository not found. Using public repositories." diff --git a/coin/provisioning/common/shared/testserver/squid/Dockerfile b/coin/provisioning/common/shared/testserver/squid/Dockerfile index f0ed68b3..330d3929 100644 --- a/coin/provisioning/common/shared/testserver/squid/Dockerfile +++ b/coin/provisioning/common/shared/testserver/squid/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 -ARG packages="squid avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon squid" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 3128-3130 diff --git a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile index 208feec7..239ea4c8 100644 --- a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="squid avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon squid" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 3128-3130 diff --git a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile index af6cd7c7..4c2e00e4 100644 --- a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile +++ b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile @@ -1,7 +1,7 @@ -FROM ubuntu:16.04 -ARG packages="vsftpd ftp wget avahi-daemon" +FROM qt_ubuntu_16.04 +ARG packages="avahi-daemon vsftpd ftp" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 20-21 # install configurations and test data -RUN wget https://tools.ietf.org/rfc/rfc3252.txt +COPY rfc3252.txt . diff --git a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile index 843472fd..8886308c 100644 --- a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile +++ b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 -ARG packages="vsftpd avahi-daemon" +FROM qt_ubuntu_18.04 +ARG packages="avahi-daemon vsftpd" RUN apt-get update && apt-get install -y $packages && dpkg -l $packages EXPOSE 20-21 diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh index aca5dd04..9f74b32e 100644 --- a/coin/provisioning/common/unix/common.sourced.sh +++ b/coin/provisioning/common/unix/common.sourced.sh @@ -43,6 +43,7 @@ export PROVISIONING_ARCH export CMD_PKG_INSTALL export CMD_PKG_LOCALINSTALL export CMD_INSTALL +export COIN_RUNS_IN_QT_COMPANY @@ -143,6 +144,13 @@ set_common_environment () { esac CMD_INSTALL="sudo install" + + COIN_RUNS_IN_QT_COMPANY=false + if ping -c1 repo-clones.ci.qt.io >/dev/null 2>&1 + then + COIN_RUNS_IN_QT_COMPANY=true + fi + } set_common_environment |