diff options
Diffstat (limited to 'coin/provisioning/common/unix')
-rwxr-xr-x | coin/provisioning/common/unix/DownloadURL.sh | 113 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/InstallFromCompressedFileFromURL.sh | 3 | ||||
-rw-r--r-- | coin/provisioning/common/unix/common.sourced.sh | 167 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/emsdk.sh | 93 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/install-openssl.sh | 6 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/install_protobuff.sh | 2 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/libclang.sh | 4 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/python_modules.sh | 38 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/squishInstall.sh | 165 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/telegraf_install.sh | 98 | ||||
-rwxr-xr-x | coin/provisioning/common/unix/telegraf_password.sh | 60 |
11 files changed, 611 insertions, 138 deletions
diff --git a/coin/provisioning/common/unix/DownloadURL.sh b/coin/provisioning/common/unix/DownloadURL.sh index 996c99da..1d48a565 100755 --- a/coin/provisioning/common/unix/DownloadURL.sh +++ b/coin/provisioning/common/unix/DownloadURL.sh @@ -1,8 +1,8 @@ -#!/usr/bin/env bash +#!/bin/sh ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -33,31 +33,94 @@ ## ############################################################################# + # A helper script used for downloading a file from a URL or an alternative -# URL. Also the SHA1 is checked for the file. Target filename should also -# be given. -# -# If called directly from another script, it will exit the parent script -# as well, if not called in its own subshell with parentheses. +# URL. Also the SHA is checked for the file (SHA algorithm is autodetected +# based on the SHA length). Target filename should also be given. -set -ex +############################ BOILERPLATE ############################### -function DownloadURL { - url=$1 - url_alt=$2 - expectedSha1=$3 - targetFile=$4 - - echo "Downloading from primary URL '$url'" - curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" || { - echo "Failed to download '$url' multiple times" - echo "Downloading from alternative URL '$url_alt'" - curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url_alt" || { echo 'Failed to download even from alternative url'; exit 1; } - } - - echo "Checking SHA1 on PKG '$targetFile'" - echo "$expectedSha1 *$targetFile" > "$targetFile.sha1" - sha1sum --check "$targetFile.sha1" - rm -f "$targetFile.sha1" +command -v sha1sum >/dev/null || sha1sum () { shasum -a 1 "$@" ; } +command -v sha256sum >/dev/null || sha256sum () { shasum -a 256 "$@" ; } +command -v sha384sum >/dev/null || sha384sum () { shasum -a 384 "$@" ; } +command -v sha512sum >/dev/null || sha512sum () { shasum -a 512 "$@" ; } + +######################################################################## + + +Download () { + url="$1" + targetFile="$2" + + command -v curl >/dev/null \ + && curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" \ + || wget --tries 5 -O "$targetFile" "$url" +} + +VerifyHash () { + file=$1 + expectedHash=$2 + + if [ ! -f "$file" ] + then return 2 # file does not exist + fi + + + hashLength="$(echo "$expectedHash" | wc -c | sed 's/ *//g')" + # Use backticks because of bug with bash-3 (default on macOS), + # caused when there are unbalanced parentheses inside $() + # shellcheck disable=SC2006 + hash=`case "$hashLength" in + 41) sha1sum "$file" ;; + 65) sha256sum "$file" ;; + 97) sha384sum "$file" ;; + 129) sha512sum "$file" ;; + *) echo "FATAL! Unknown hash length: $hashLength" 1>&2 && exit 1 ;; + esac | cut -d ' ' -f 1` + + if [ ! "$expectedHash" = "$hash" ] + then + echo "FAIL! wrong file hash: $file $hash" 1>&2 + return 1 + fi + echo "OK verified integrity of: $file" } +# Check if file already exists and is good, otherwise download it +DownloadURL () { + url=$1 + url2=$2 + expectedHash=$3 + # Optional argument $4: destination filename + if [ x"$4" = x ] + then + # defaults to the last component of $url + targetFile=$(echo $url | sed 's|^.*/||') + else + targetFile=$4 + fi + + # If a non-empty file already exists + if [ -s "$targetFile" ] + then + if VerifyHash "$targetFile" "$expectedHash" + then + echo "Skipping download, found and validated existing file: $targetFile" + return + else + echo "WARNING: Non-empty but different file found at destination; will re-download and overwrite file: $targetFile" + fi + fi + + echo "Downloading from primary URL: $url" + if ! Download "$url" "$targetFile" + then + echo "FAIL! to download, trying alternative URL: $url2" 1>&2 + if ! Download "$url2" "$targetFile" + then + echo 'FAIL! to download even from alternative URL' 1>&2 + return 1 + fi + fi + VerifyHash "$targetFile" "$expectedHash" +} diff --git a/coin/provisioning/common/unix/InstallFromCompressedFileFromURL.sh b/coin/provisioning/common/unix/InstallFromCompressedFileFromURL.sh index 34bcf289..8aaa0dc3 100755 --- a/coin/provisioning/common/unix/InstallFromCompressedFileFromURL.sh +++ b/coin/provisioning/common/unix/InstallFromCompressedFileFromURL.sh @@ -67,6 +67,9 @@ function InstallFromCompressedFileFromURL { "xz") tar -xf "$targetFile" --directory "$targetDirectory" ;; + "tbz2") + tar -xjf "$targetFile" --directory "$targetDirectory" + ;; *) exit 1 ;; diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh new file mode 100644 index 00000000..9f74b32e --- /dev/null +++ b/coin/provisioning/common/unix/common.sourced.sh @@ -0,0 +1,167 @@ +#!/bin/sh + + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +# Script to be sourced from everywhere you need a common environment. Defines: +export PROVISIONING_DIR +export PROVISIONING_OS +export PROVISIONING_OS_ID +export PROVISIONING_ARCH +export CMD_PKG_INSTALL +export CMD_PKG_LOCALINSTALL +export CMD_INSTALL +export COIN_RUNS_IN_QT_COMPANY + + + +if [ x"$IS_PROVISIONING_COMMON_SOURCED" != x ] +then + echo "common.sourced.sh has already been sourced, re-sourcing skipped" + return +fi + +# Do not export; you want children to re-source, because they only inherit the +# variables but not the functions +IS_PROVISIONING_COMMON_SOURCED=1 + + +fatal () { + echo "$1" 1>&2 + if [ x"$2" != x ] + then exit "$2" + else exit + fi +} + +# Takes one argument which should be the filename of this script. Returns true +# if the script is being sourced, false if the script is being executed. +is_script_executed () { + [ x"$(basename $(echo "$0" | sed s/^-//))" = x"$1" ] +} + + +is_script_executed common.sourced.sh \ + && fatal "Script common.sourced.sh should always be sourced, not executed" + + +_detect_linux_OS_ID () { + if [ -f /etc/os-release ] + then + . /etc/os-release + PROVISIONING_OS_ID="$ID" + elif [ -f /etc/redhat-release ] + then + case "$(cat /etc/redhat-release)" in + "Red Hat Enterprise Linux"*) + PROVISIONING_OS_ID="rhel" + ;; + "CentOS Linux"*) + PROVISIONING_OS_ID="centos" + ;; + *) fatal "Unknown string in /etc/redhat-release" ;; + esac + fi +} + +set_common_environment () { + # Unfortunately we can't find the provisioning directory from a sourced + # script in a portable way + # PROVISIONING_DIR="$(dirname "$0")/../../" + + [ x"$PROVISIONING_DIR" = x ] \ + && fatal "PROVISIONING_DIR variable needs to be set before calling set_common_environment" + + uname_s="$(uname -s)" + case "$uname_s" in + Linux) + PROVISIONING_OS=linux + _detect_linux_OS_ID + case "$PROVISIONING_OS_ID" in + suse|sles|opensuse*) + CMD_PKG_INSTALL="sudo zypper -nq install" + CMD_PKG_LOCALINSTALL="sudo zypper --no-gpg-checks -nq install" + ;; + debian|ubuntu) + CMD_PKG_INSTALL="sudo apt -y install" + CMD_PKG_LOCALINSTALL="sudo apt -y install" + ;; + rhel|centos|fedora) + CMD_PKG_INSTALL="sudo yum -y install" + CMD_PKG_LOCALINSTALL="sudo yum -y --nogpgcheck localinstall" + ;; + *) fatal "Unknown ID in /etc/os-release: $PROVISIONING_OS_ID" ;; + esac + ;; + Darwin) + PROVISIONING_OS=macos + PROVISIONING_OS_ID=macos + CMD_PKG_INSTALL="brew install" + CMD_PKG_LOCALINSTALL="echo 'TODO how to install a package file on macOS'" + ;; + *) + fatal "Unknown system in uname: $uname_s" 42 + ;; + esac + + uname_m="$(uname -m)" + case "$uname_m" in + x86_64|amd64) PROVISIONING_ARCH=amd64 ;; + i[3456]86) PROVISIONING_ARCH=x86 ;; + *) fatal "Unknown architecture in uname: $uname_m" 43 ;; + esac + + CMD_INSTALL="sudo install" + + COIN_RUNS_IN_QT_COMPANY=false + if ping -c1 repo-clones.ci.qt.io >/dev/null 2>&1 + then + COIN_RUNS_IN_QT_COMPANY=true + fi + +} + +set_common_environment + +set_dry_run () { + if [ x"$PROVISIONING_DRY_RUN" != x ] + then + CMD_PKG_INSTALL="echo DRYRUN: $CMD_PKG_INSTALL" + CMD_PKG_LOCALINSTALL="echo DRYRUN: $CMD_PKG_LOCALINSTALL" + CMD_INSTALL="echo DRYRUN: $CMD_INSTALL" + fi +} + +set_dry_run diff --git a/coin/provisioning/common/unix/emsdk.sh b/coin/provisioning/common/unix/emsdk.sh new file mode 100755 index 00000000..a2fda735 --- /dev/null +++ b/coin/provisioning/common/unix/emsdk.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +# shellcheck source=./InstallFromCompressedFileFromURL.sh +source "${BASH_SOURCE%/*}/InstallFromCompressedFileFromURL.sh" +# shellcheck source=./SetEnvVar.sh +source "${BASH_SOURCE%/*}/SetEnvVar.sh" +# shellcheck source=./DownloadURL.sh +source "${BASH_SOURCE%/*}/DownloadURL.sh" + +version="1.39.8" +urlCache="http://ci-files01-hki.intra.qt.io/input/emsdk" +targetFolder="/opt/emsdk" + +# cross-platform emscripten SDK +urlEmscriptenExternal="https://github.com/emscripten-core/emscripten/archive/$version.tar.gz" +urlEmscriptenCache="$urlCache/emscripten.$version.tar.gz" +sha1Emscripten="a593ea3b4ab7e3d57e1232b19a903ccf8f137d2f" + +# platform-specific toolchain and node binaries. urls obtained from "emsdk install" +if uname -a |grep -q Darwin; then + urlWasmBinariesExternal="https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/9e60f34accb4627d7358223862a7e74291886ab6/wasm-binaries.tbz2" + urlWasmBinariesCache="$urlCache/macos/wasm-binaries.$version.tbz2" + sha1WasmBinaries="aedb30fb07d565c35305af0920ab072ae743895d" + + urlNodeBinariesExternal="https://storage.googleapis.com/webassembly/emscripten-releases-builds/deps/node-v12.9.1-darwin-x64.tar.gz" + urlNodeBinariesCache="$urlCache/mac/node-v12.9.1-darwin-x64.tar.gz" + sha1NodeBinaries="f5976321ded091e70358e406b223f6fd64e35a43" + pathNodeExecutable='node-v12.9.1-darwin-x64/bin/node' +else + urlWasmBinariesExternal="https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/9e60f34accb4627d7358223862a7e74291886ab6/wasm-binaries.tbz2" + urlWasmBinariesCache="$urlCache/linux/wasm-binaries.$version.tbz2" + sha1WasmBinaries="eb7fc94aa79a6e215272e2586173515aa37c3141" + + urlNodeBinariesExternal="https://storage.googleapis.com/webassembly/emscripten-releases-builds/deps/node-v12.9.1-linux-x64.tar.xz" + urlNodeBinariesCache="$urlCache/linux/node-v12.9.1-linux-x64.tar.xz" + sha1NodeBinaries="cde96023b468d593c50de27470dd714c8cfda9aa" + pathNodeExecutable='node-v12.9.1-linux-x64/bin/node' +fi + +sudo mkdir "$targetFolder" + +InstallFromCompressedFileFromURL "$urlEmscriptenCache" "$urlEmscriptenExternal" "$sha1Emscripten" "$targetFolder" "" +InstallFromCompressedFileFromURL "$urlWasmBinariesCache" "$urlWasmBinariesExternal" "$sha1WasmBinaries" "$targetFolder" "" +InstallFromCompressedFileFromURL "$urlNodeBinariesCache" "$urlNodeBinariesExternal" "$sha1NodeBinaries" "$targetFolder" "" + +sudo chmod -R a+rwx "$targetFolder" + +echo "Writing $targetFolder/.emscripten" +cat <<EOM >"$targetFolder/.emscripten" +EMSCRIPTEN_ROOT='$targetFolder/emscripten-$version' +LLVM_ROOT='$targetFolder/install/bin' +BINARYEN_ROOT='$targetFolder/install' +NODE_JS='$targetFolder/$pathNodeExecutable' +TEMP_DIR = '/tmp' +EOM + +SetEnvVar "PATH" "\"\$PATH:$targetFolder/emscripten-$version/\"" +SetEnvVar "EMSCRIPTEN" "$targetFolder/emscripten-$version" +SetEnvVar "EM_CONFIG" "$targetFolder/.emscripten" + +echo "Emsdk = $version" >> ~/versions.txt diff --git a/coin/provisioning/common/unix/install-openssl.sh b/coin/provisioning/common/unix/install-openssl.sh index 01a20935..75b7edbc 100755 --- a/coin/provisioning/common/unix/install-openssl.sh +++ b/coin/provisioning/common/unix/install-openssl.sh @@ -2,7 +2,7 @@ ############################################################################# ## -## Copyright (C) 2019 The Qt Company Ltd. +## Copyright (C) 2020 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -42,11 +42,11 @@ source "${BASH_SOURCE%/*}/../unix/DownloadURL.sh" # shellcheck source=../unix/SetEnvVar.sh source "${BASH_SOURCE%/*}/../unix/SetEnvVar.sh" -version="1.1.1b" +version="1.1.1g" officialUrl="https://www.openssl.org/source/openssl-$version.tar.gz" cachedUrl="http://ci-files01-hki.intra.qt.io/input/openssl/openssl-$version.tar.gz" targetFile="/tmp/openssl-$version.tar.gz" -sha="e9710abf5e95c48ebf47991b10cbb48c09dae102" +sha="b213a293f2127ec3e323fb3cfc0c9807664fd997" opensslHome="${HOME}/openssl-${version}" opensslSource="${opensslHome}-src" DownloadURL "$cachedUrl" "$officialUrl" "$sha" "$targetFile" diff --git a/coin/provisioning/common/unix/install_protobuff.sh b/coin/provisioning/common/unix/install_protobuff.sh index 59e9b699..a9b2cfd0 100755 --- a/coin/provisioning/common/unix/install_protobuff.sh +++ b/coin/provisioning/common/unix/install_protobuff.sh @@ -64,7 +64,7 @@ if uname -a |grep -q Darwin; then else ./configure fi -make +make -j5 sudo make install # Refresh shared library cache if OS isn't macOS diff --git a/coin/provisioning/common/unix/libclang.sh b/coin/provisioning/common/unix/libclang.sh index a9d1cc49..6fa6c040 100755 --- a/coin/provisioning/common/unix/libclang.sh +++ b/coin/provisioning/common/unix/libclang.sh @@ -65,10 +65,10 @@ zip="/tmp/libclang.7z" destination="/usr/local/libclang-$version" DownloadURL $url_cached $url $sha1 $zip -7z x $zip -o/tmp/ +sudo 7z x $zip -o/usr/local/ +sudo mv /usr/local/libclang "$destination" rm -rf $zip -sudo mv /tmp/libclang "$destination" echo "export LLVM_INSTALL_DIR=$destination" >> ~/.bash_profile echo "libClang = $version" >> ~/versions.txt diff --git a/coin/provisioning/common/unix/python_modules.sh b/coin/provisioning/common/unix/python_modules.sh new file mode 100755 index 00000000..89adbd5c --- /dev/null +++ b/coin/provisioning/common/unix/python_modules.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +# needed by packaging scripts +pip3 install bs4 +pip3 install sh diff --git a/coin/provisioning/common/unix/squishInstall.sh b/coin/provisioning/common/unix/squishInstall.sh index b40c1922..7592814b 100755 --- a/coin/provisioning/common/unix/squishInstall.sh +++ b/coin/provisioning/common/unix/squishInstall.sh @@ -2,7 +2,7 @@ ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2020 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -33,120 +33,79 @@ ## ############################################################################# +# shellcheck source=./DownloadURL.sh +source "${BASH_SOURCE%/*}/DownloadURL.sh" + set -ex -# This script will install squish package for Linux and Mac. +# This script will fetch and extract pre-buildt squish package for Linux and Mac. # Squish is need by Release Test Automation (RTA) -version="6.4.3" -# Branch version without dot -qtBranch="512x" -squishFolder="/opt/squish" -squishUrl="ci-files01-hki.intra.qt.io:/hdd/www/input/squish/coin/$qtBranch/" -squishFile="squish-$version-qt$qtBranch-linux64.run" +version="6.5.2" +qtBranch="514x" +installFolder="/opt" +squishFolder="$installFolder/squish" +preBuildCacheUrl="ci-files01-hki.intra.qt.io:/hdd/www/input/squish/jenkins_build/stable" +licenseUrl="http://ci-files01-hki.intra.qt.io/input/squish/coin/.squish-3-license" if uname -a |grep -q Darwin; then - squishFile="squish-$version-qt$qtBranch-macx86_64.dmg" + compressedFolder="prebuild-squish-$version-$qtBranch-macx86_64.tar.gz" + sha1="91a1d7228dc4b4bd64f93191c33d2642abc97e14" +else + compressedFolder="prebuild-squish-$version-$qtBranch-linux64.tar.gz" + sha1="63f33de55770e2588dd9f482546900e33d20f8ec" fi -squishLicenseUrl="ci-files01-hki.intra.qt.io:/hdd/www/input/squish/coin/" -squishLicenseFile=".squish-3-license.tar.gz" - -testSuite="suite_test_squish" -testSuiteUrl="ci-files01-hki.intra.qt.io:/hdd/www/input/squish/coin/" +mountFolder="/tmp/squish" +sudo mkdir "$mountFolder" -# These checks can be removed when Vanilla OS for all linux and Mac are in -if [ -d "$squishFolder" ]; then - echo "Move old squish to /tmp" - sudo mv "$squishFolder" "/tmp/squish_$(date)" +# Check which platform +if uname -a |grep -q Darwin; then + usersGroup="staff" + squishLicenseDir="/Users/qt" +elif uname -a |grep -q "el7"; then + usersGroup="qt" + squishLicenseDir="/root" +elif uname -a |grep -q "Ubuntu"; then + usersGroup="users" + squishLicenseDir="/home/qt" +else + usersGroup="users" + squishLicenseDir="/root" fi -if [ -f "/etc/profile.d/squish_env.sh" ]; then - echo "Remove /etc/profile.d/squish_env.sh" - sudo rm -f "/etc/profile.d/squish_env.sh" - export SQUISH_LICENSEKEY_DIR=$HOME +targetFileMount="$mountFolder"/"$compressedFolder" + +echo "Mounting $preBuildCacheUrl to $mountFolder" +sudo mount "$preBuildCacheUrl" "$mountFolder" +echo "Create $installFolder if needed" +if [ ! -d "$installFolder" ]; then + sudo mkdir "$installFolder" fi -function MountAndInstall { - url=$1 - targetDirectory=$2 - targetFile=$3 - - # Check which platform - if uname -a |grep -q Darwin; then - usersGroup="staff" - mountFolder="/Volumes" - squishLicenseDir="/Users/qt" - elif uname -a |grep -q "el7"; then - usersGroup="qt" - mountFolder="/tmp" - squishLicenseDir="/root" - elif uname -a |grep -q "Ubuntu"; then - usersGroup="users" - mountFolder="/tmp" - squishLicenseDir="/home/qt" - else - usersGroup="users" - mountFolder="/tmp" - squishLicenseDir="/root" - fi +VerifyHash "$targetFileMount" "$sha1" - function UnMount { - echo "Unmounting $mountFolder" - sudo diskutil unmount force "$mountFolder" || sudo umount -f "$mountFolder" - } +echo "Uncompress $compressedFolder" +sudo tar -xzf "$targetFileMount" --directory "$installFolder" - targetFileMount="$mountFolder"/"$targetFile" +echo "Unmounting $mountFolder" +sudo diskutil unmount force "$mountFolder" || sudo umount -f "$mountFolder" - echo "Mounting $url to $mountFolder" - sudo mount "$url" "$mountFolder" - echo "Create $targetDirectory if needed" - if [ ! -d "/opt" ]; then - sudo mkdir "/opt" - fi - if [ ! -d "$targetDirectory" ]; then - sudo mkdir "$targetDirectory" - fi - echo "Uncompress $targetFile" - if [[ $targetFile == *.tar.gz ]]; then - if [[ $targetFile == .squish-3-license.* ]]; then - target="$squishLicenseDir" - # Squish license need to be exists also in users home directory, because squish check it before it starts running tests - sudo tar -xzf "$targetFileMount" --directory "$HOME" - else - target="$targetDirectory" - fi - sudo tar -xzf "$targetFileMount" --directory "$target" - UnMount - elif [[ $targetFile == *.dmg ]]; then - echo "'dmg-file', no need to uncompress" - sudo cp $targetFileMount /tmp - UnMount - sudo hdiutil attach "/tmp/$targetFile" - sudo /Volumes/froglogic\ Squish/Install\ Squish.app/Contents/MacOS/Squish unattended=1 targetdir="$targetDirectory/package" qtpath="$targetDirectory" > /dev/null 2>&1 - mountFolder="/Volumes/froglogic Squish" - UnMount - elif [[ $targetFile == *.run ]]; then - echo "'run-file', no need to uncompress" - sudo cp $targetFileMount $targetDirectory - UnMount - sudo chmod +x $targetDirectory/$targetFile - sudo $targetDirectory/$targetFile unattended=1 targetdir="$targetDirectory/package" qtpath="$targetDirectory" > /dev/null 2>&1 - sudo rm -fr "$targetDirectory/$targetFile" - if uname -a |grep -q "Ubuntu"; then - sudo mkdir /usr/lib/tcl8.6 - sudo cp "$targetDirectory/package/tcl/lib/tcl8.6/init.tcl" /usr/lib/tcl8.6/ - fi - else - exit 1 +sudo mv "$installFolder/rta_squish_$version" "$squishFolder" + +if uname -a |grep -q "Ubuntu"; then + if [ ! -e "/usr/lib/tcl8.6" ]; then + sudo mkdir /usr/lib/tcl8.6 + sudo cp "$squishFolder/package/tcl/lib/tcl8.6/init.tcl" /usr/lib/tcl8.6/ fi +fi - echo "Changing ownerships" - sudo chown -R qt:$usersGroup "$targetDirectory" - sudo chown qt:$usersGroup "$HOME/.squish-3-license" -} +DownloadURL "$licenseUrl" "$licenseUrl" "3c6b2b3ca047f68e20f697afcc3a9d376f01b376" "$HOME/.squish-3-license" -echo "Set commands for environment variables in .bashrc" +echo "Changing ownerships" +sudo chown -R qt:$usersGroup "$squishFolder" +sudo chown qt:$usersGroup "$HOME/.squish-3-license" +echo "Set commands for environment variables in .bashrc" if uname -a |grep -q "Ubuntu"; then echo "export SQUISH_PATH=$squishFolder/package" >> ~/.profile echo "export PATH=\$PATH:$squishFolder/squish-$version/bin" >> ~/.profile @@ -155,19 +114,11 @@ else echo "export PATH=\$PATH:$squishFolder/squish-$version/bin" >> ~/.bashrc fi -echo "Installing squish license to home directory.." -MountAndInstall "$squishLicenseUrl" "$squishFolder" "$squishLicenseFile" - -echo "Installing squish $version.." -MountAndInstall "$squishUrl" "$squishFolder" "$squishFile" - -echo "Installing provisioning scripts for squish" -MountAndInstall "$testSuiteUrl" "$squishFolder" "$testSuite.tar.gz" - -echo "Verifying Squish Installation" -if "$squishFolder/package/bin/squishrunner" --testsuite "$squishFolder/$testSuite" | grep "Squish test run successfully" ; then +echo "Verifying Squish" +if "$squishFolder/package/bin/squishrunner" --testsuite "$squishFolder/suite_test_squish" | grep "Squish test run successfully" ; then echo "Squish installation tested successfully" else echo "Squish test failed! Package wasn't installed correctly." exit 1 fi + diff --git a/coin/provisioning/common/unix/telegraf_install.sh b/coin/provisioning/common/unix/telegraf_install.sh new file mode 100755 index 00000000..b2dd244e --- /dev/null +++ b/coin/provisioning/common/unix/telegraf_install.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +# This script installs telegraf and ioping and our script telegraf-ioping.sh +# to the /usr/bin directory. +# +# The reasons we don't install to /usr/local/bin are: +# 1. On SLES and RHEL, the PATH of sudo (secure_path setting in /etc/sudoers) +# does not include /usr/local/bin. +# 2. On macOS /usr/local/bin does not even exist early in provisioning. + + +######################## BOILERPLATE ########################### +set -e + + +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh + +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh + +is_script_executed telegraf_install.sh \ + || fatal "Script telegraf_install.sh should be executed, not sourced" + +################################################################ + + +[ "$PROVISIONING_OS" = linux ] \ + && ioping_sha256=259abf04bcb84f4126ff97c04b6651e1cf5ea6d8a9ff364c769a26c95b6eeb44 \ + || ioping_sha256=55de6a2f1a5343e0ce8df31d82d47a9e79c7e612edbc6dfb39b5fc6fb358b2e3 +DownloadURL "http://ci-files01-hki.intra.qt.io/input/ioping/ioping.${PROVISIONING_OS}-${PROVISIONING_ARCH}" \ + '' "$ioping_sha256" ioping +$CMD_INSTALL -m 755 ./ioping /usr/bin/ +rm -f ioping + +# 2. Install custom ioping monitoring script +$CMD_INSTALL -m 755 "$PROVISIONING_DIR"/common/shared/telegraf/telegraf-ioping.sh /usr/bin/ + +# 3. Download and install telegraf + +[ x"$PROVISIONING_OS" = xmacos ] && os=darwin || os=linux +[ x"$PROVISIONING_ARCH" = xx86 ] && arch=i386 || arch=amd64 +package_filename=telegraf-1.12.6_${os}_${arch}.tar.gz +package_sha256_list="$PROVISIONING_DIR"/common/shared/telegraf/telegraf_packages.sha256.txt +package_sha256=$(sed -n "s/.*$package_filename *//p" "$package_sha256_list") + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/telegraf/"$package_filename" \ + https://dl.influxdata.com/telegraf/releases/"$package_filename" \ + "$package_sha256" \ + telegraf.tar.gz + +tar -xzf ./telegraf.tar.gz -C /tmp +telegraf_binary=$(find /tmp/telegraf* -name telegraf -type f | grep /bin/ | head -1) +$CMD_INSTALL -m 755 "$telegraf_binary" /usr/bin/ +rm -rf /tmp/telegraf* + +# 4. Edit config file with passwords +"$PROVISIONING_DIR"/common/unix/telegraf_password.sh + +# 5. Start telegraf in background (-b) and with retaining the environment (-E) +# in order to report as hostname = $COIN_UNIQUE_JOB_ID. +/usr/bin/sudo -b -E telegraf --config /etc/telegraf-coin.conf >/dev/null 2>&1 + +echo DONE: "Installed and started telegraf: $package_filename" diff --git a/coin/provisioning/common/unix/telegraf_password.sh b/coin/provisioning/common/unix/telegraf_password.sh new file mode 100755 index 00000000..efbed142 --- /dev/null +++ b/coin/provisioning/common/unix/telegraf_password.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +######################## BOILERPLATE ########################### +set -e + + +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh + +is_script_executed telegraf_password.sh \ + || fatal "Script telegraf_password.sh should be executed, not sourced, to avoid leaking secrets in the logs" +# Avoid leaking secrets in the logs +set +x +################################################################ + + +# Provisioning should run even without the secrets repository +influxdb_password=$(cut -d : -f 2 ~qt/work/influxdb/coin_vms_writer.auth) \ + || influxdb_password=no_password_provided + +rm -f ~qt/work/influxdb/coin_vms_writer.auth +sed "s|COIN_VMS_WRITER_PASS|$influxdb_password|" \ + "$PROVISIONING_DIR"/common/"$PROVISIONING_OS"/telegraf-coin.conf \ + > .telegraf-coin.conf.final +$CMD_INSTALL -m 600 .telegraf-coin.conf.final /etc/telegraf-coin.conf +rm -f .telegraf-coin.conf.final |