diff options
author | Ryan Chu <ryan.chu@qt.io> | 2018-10-01 10:38:39 +0200 |
---|---|---|
committer | Liang Qi <liang.qi@qt.io> | 2018-10-30 08:15:43 +0000 |
commit | ea76ace8257d98a8c6d72fca39b1f7f559f084cb (patch) | |
tree | 443a6b56de42052ecca1c3c87c31bfefd53fb3fb /coin/provisioning/common | |
parent | a76e315034d12f85a7a5281f1384375d2fb4ae3b (diff) |
Docker Provisioning: Install Docker-based test servers on macOS
Build and cache docker images based on the configurations in testserver.
1. Move the shared Docker files to common/shared/testserver
2. Deploy boot2docker image into VirtualBox via docker-machine commands.
3. Build and tag the docker images by the SHA-1 of server context.
Change-Id: Ic5a588e5cc5e753ec4c4e2e1bc79917bbd03f95d
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
Diffstat (limited to 'coin/provisioning/common')
14 files changed, 273 insertions, 9 deletions
diff --git a/coin/provisioning/common/linux/docker.sh b/coin/provisioning/common/linux/docker.sh index 16af0a0a..07cedba9 100755 --- a/coin/provisioning/common/linux/docker.sh +++ b/coin/provisioning/common/linux/docker.sh @@ -2,7 +2,7 @@ ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -49,4 +49,4 @@ sudo curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-c sudo chmod +x /usr/local/bin/docker-compose # Start testserver provisioning -source "${BASH_SOURCE%/*}/testserver/docker_testserver.sh" +source "${BASH_SOURCE%/*}/docker_testserver.sh" diff --git a/coin/provisioning/common/linux/testserver/docker_testserver.sh b/coin/provisioning/common/linux/docker_testserver.sh index a88e89b9..a272d706 100755 --- a/coin/provisioning/common/linux/testserver/docker_testserver.sh +++ b/coin/provisioning/common/linux/docker_testserver.sh @@ -2,7 +2,7 @@ ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -35,24 +35,33 @@ set -ex +case ${BASH_SOURCE[0]} in + */linux/*) SERVER_PATH="${BASH_SOURCE[0]%/linux/*}/shared/testserver" ;; + */*) SERVER_PATH="${BASH_SOURCE[0]%/*}/../shared/testserver" ;; + *) SERVER_PATH="../shared/testserver" ;; +esac + +# testserver shared scripts +source "$SERVER_PATH/testserver_util.sh" + # Using SHA-1 of each server context as the tag of docker images. A tag labels a # specific image version. It is used by docker compose file (docker-compose.yml) # to launch the corresponding docker containers. If one of the server contexts # (./apache2, ./danted, ...) gets changes, all the related compose files in # qtbase should be updated as well. -# -# For example, here's how to get the SHA-1 of apache test server. -# find ./apache2 -type f -print0 | sort -z | xargs -r0 sha1sum | \ -# awk '{ print $1 }' | sha1sum | awk '{ print $1 }' -testserver='apache2 squid vsftpd ftp-proxy danted' +source "$SERVER_PATH/settings.sh" for server in $testserver do - context="${BASH_SOURCE%/*}/$server" + context="$SERVER_PATH/$server" + # obsolete - This one is still needed for CI. It will be removed later. sha1=$(find $context -type f -print0 | sort -z | xargs -r0 sha1sum | awk '{ print $1 }' | \ sha1sum | awk '{ print $1 }') sudo docker build -t qt-test-server-$server:$sha1 $context + + # Sort files by their SHA-1 and use the accumulated result as the TAG + sudo docker build -t qt-test-server-$server:$(sha1tree $context) $context done sudo docker images diff --git a/coin/provisioning/common/macos/docker.sh b/coin/provisioning/common/macos/docker.sh new file mode 100755 index 00000000..96265454 --- /dev/null +++ b/coin/provisioning/common/macos/docker.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +set -ex + +# Download and install the Docker Toolbox for macOS (Docker Compose and Docker Machine). +url="https://download.docker.com/mac/stable/DockerToolbox.pkg" +target_file="DockerToolbox.pkg" + +if [ -x "$(command -v sha1sum)" ] +then + # This part shall be used in CI environment only. The DownloadURL script needs sha1sum + # which is not included in the default macOS system. In addition, the cached pkg can't + # be downloaded out of the Qt internal network. + case ${BASH_SOURCE[0]} in + */macos/*) UNIX_PATH="${BASH_SOURCE[0]%/macos/*}/unix" ;; + */*) UNIX_PATH="${BASH_SOURCE[0]%/*}/../unix" ;; + *) UNIX_PATH="../unix" ;; + esac + + source "$UNIX_PATH/DownloadURL.sh" + url_cached="http://ci-files01-hki.ci.local/input/windows/DockerToolbox.pkg" + sha1="7196d2d30648d486978d29adb5837ff7876517c1" + DownloadURL $url_cached $url $sha1 $target_file +else + curl $url -o $target_file +fi +sudo installer -pkg $target_file -target / + +# Start testserver provisioning +source "${BASH_SOURCE%/*}/docker_testserver.sh" diff --git a/coin/provisioning/common/macos/docker_testserver.sh b/coin/provisioning/common/macos/docker_testserver.sh new file mode 100755 index 00000000..0247921f --- /dev/null +++ b/coin/provisioning/common/macos/docker_testserver.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +set -ex + +case ${BASH_SOURCE[0]} in + */macos/*) SERVER_PATH="${BASH_SOURCE[0]%/macos/*}/shared/testserver" ;; + */*) SERVER_PATH="${BASH_SOURCE[0]%/*}/../shared/testserver" ;; + *) SERVER_PATH="../shared/testserver" ;; +esac + +# testserver shared scripts +source "$SERVER_PATH/testserver_util.sh" + +# Create docker virtual machine (Boot2docker) +source "$SERVER_PATH/docker_machine.sh" + +# Using SHA-1 of each server context as the tag of docker images. A tag labels a +# specific image version. It is used by docker compose file (docker-compose.yml) +# to launch the corresponding docker containers. If one of the server contexts +# (./apache2, ./danted, ...) gets changes, all the related compose files in +# qtbase should be updated as well. + +source "$SERVER_PATH/settings.sh" + +for server in $testserver +do + context="$SERVER_PATH/$server" + docker build -t qt-test-server-$server:$(sha1tree $context) $context +done + +docker images diff --git a/coin/provisioning/common/linux/testserver/apache2/Dockerfile b/coin/provisioning/common/shared/testserver/apache2/Dockerfile index bb2dc850..bb2dc850 100644 --- a/coin/provisioning/common/linux/testserver/apache2/Dockerfile +++ b/coin/provisioning/common/shared/testserver/apache2/Dockerfile diff --git a/coin/provisioning/common/linux/testserver/danted/Dockerfile b/coin/provisioning/common/shared/testserver/danted/Dockerfile index eb2ac6ff..eb2ac6ff 100644 --- a/coin/provisioning/common/linux/testserver/danted/Dockerfile +++ b/coin/provisioning/common/shared/testserver/danted/Dockerfile diff --git a/coin/provisioning/common/linux/testserver/danted/danted b/coin/provisioning/common/shared/testserver/danted/danted index 1da8203e..1da8203e 100755 --- a/coin/provisioning/common/linux/testserver/danted/danted +++ b/coin/provisioning/common/shared/testserver/danted/danted diff --git a/coin/provisioning/common/linux/testserver/danted/danted-authenticating b/coin/provisioning/common/shared/testserver/danted/danted-authenticating index 7aa35db6..7aa35db6 100755 --- a/coin/provisioning/common/linux/testserver/danted/danted-authenticating +++ b/coin/provisioning/common/shared/testserver/danted/danted-authenticating diff --git a/coin/provisioning/common/shared/testserver/docker_machine.sh b/coin/provisioning/common/shared/testserver/docker_machine.sh new file mode 100755 index 00000000..601056a4 --- /dev/null +++ b/coin/provisioning/common/shared/testserver/docker_machine.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +set -ex + +TestMachine='qt-test-server' + +# Deploy docker virtual machine (Boot2Docker) into VirtualBox only if it doesn't exist +if [ -z $(docker-machine ls -q --filter "name=$TestMachine") ] +then + docker-machine create -d virtualbox $TestMachine + docker-machine ip $TestMachine +fi + +# Switch the docker engine to $TestMachine +eval $(docker-machine env $TestMachine) + +docker-machine ls diff --git a/coin/provisioning/common/linux/testserver/ftp-proxy/Dockerfile b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile index cecb2dca..cecb2dca 100644 --- a/coin/provisioning/common/linux/testserver/ftp-proxy/Dockerfile +++ b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile diff --git a/coin/provisioning/common/shared/testserver/settings.sh b/coin/provisioning/common/shared/testserver/settings.sh new file mode 100644 index 00000000..041661f2 --- /dev/null +++ b/coin/provisioning/common/shared/testserver/settings.sh @@ -0,0 +1,37 @@ +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +set -ex + +# A list of test servers to be provisioned +testserver='apache2 squid vsftpd ftp-proxy danted' diff --git a/coin/provisioning/common/linux/testserver/squid/Dockerfile b/coin/provisioning/common/shared/testserver/squid/Dockerfile index 0b930bca..0b930bca 100644 --- a/coin/provisioning/common/linux/testserver/squid/Dockerfile +++ b/coin/provisioning/common/shared/testserver/squid/Dockerfile diff --git a/coin/provisioning/common/shared/testserver/testserver_util.sh b/coin/provisioning/common/shared/testserver/testserver_util.sh new file mode 100644 index 00000000..c42fb5e0 --- /dev/null +++ b/coin/provisioning/common/shared/testserver/testserver_util.sh @@ -0,0 +1,41 @@ +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +set -ex + +# Sort files by their SHA-1, and then return the accumulated result +sha1tree () { + find "$@" -type f -print0 | \ + xargs -0 shasum | cut -d ' ' -f 1 | \ + sort | shasum | cut -d ' ' -f 1 +} diff --git a/coin/provisioning/common/linux/testserver/vsftpd/Dockerfile b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile index f8e32d40..f8e32d40 100644 --- a/coin/provisioning/common/linux/testserver/vsftpd/Dockerfile +++ b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile |