From 24f0367a60fdd3379014e507fa7f4553d433c601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tony=20Saraj=C3=A4rvi?= Date: Wed, 27 Feb 2019 13:22:06 +0200 Subject: Add Suse Linux Enteprise Server 12 to the CI Task-number: QTQAINFRA-2851 Change-Id: Ia421a5e35f8a0515d7f761ca0c52a54a93dcee5c Reviewed-by: Heikki Halmet --- coin/provisioning/common/linux/disable-notifications.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/disable-notifications.sh b/coin/provisioning/common/linux/disable-notifications.sh index 7e805833..28d21656 100755 --- a/coin/provisioning/common/linux/disable-notifications.sh +++ b/coin/provisioning/common/linux/disable-notifications.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash -sudo gsettings set org.gnome.desktop.notifications.application:/update-manager/ enable false -sudo gsettings set org.gnome.desktop.notifications show-banners false +gsettings set org.gnome.desktop.notifications.application:/update-manager/ enable false +gsettings set org.gnome.desktop.notifications show-banners false -- cgit v1.2.3 From 8e32f46764ec8b06ac1ce4cc4e3a70580328ce23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tony=20Saraj=C3=A4rvi?= Date: Tue, 19 Mar 2019 12:02:57 +0200 Subject: Add Suse Linux Enteprise Server 15 to the CI Task-number: QTQAINFRA-3123 Change-Id: I725c58c7d319945f7470118dec99b5360385da7b Reviewed-by: Heikki Halmet --- coin/provisioning/common/linux/disable-ntp_linux.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/disable-ntp_linux.sh b/coin/provisioning/common/linux/disable-ntp_linux.sh index b73e44a5..4ded6042 100755 --- a/coin/provisioning/common/linux/disable-ntp_linux.sh +++ b/coin/provisioning/common/linux/disable-ntp_linux.sh @@ -39,6 +39,8 @@ echo "Disable Network Time Protocol (NTP)" if uname -a |grep -q "Ubuntu"; then sudo timedatectl set-ntp false +elif cat /etc/os-release |grep "SUSE Linux Enterprise Server 15"; then + sudo timedatectl set-ntp false else (systemctl &>/dev/null && sudo systemctl disable ntpd) || sudo /sbin/chkconfig ntpd off fi -- cgit v1.2.3 From 0d273deeef0c2ea44cd862a1df756ab99cf2926d Mon Sep 17 00:00:00 2001 From: Assam Boudjelthia Date: Wed, 4 Sep 2019 09:54:10 +0300 Subject: Provisioning: Fix version of libiodbc on MacOs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libiodbc latest version (3.52.13) was automatically installed causing qt-config-gui to fail build, this installs version 3.52.12. Task-number: QTQAINFRA-3188 Change-Id: I5b87ff04c2301a5471cc9abfa0849b6073a1bdd4 Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/macos/libiodbc.rb | 23 +++++++++++++++++++++++ coin/provisioning/common/macos/libiodbc.sh | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 coin/provisioning/common/macos/libiodbc.rb (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/macos/libiodbc.rb b/coin/provisioning/common/macos/libiodbc.rb new file mode 100644 index 00000000..ed669e69 --- /dev/null +++ b/coin/provisioning/common/macos/libiodbc.rb @@ -0,0 +1,23 @@ +class Libiodbc < Formula + desc "Database connectivity layer based on ODBC. (alternative to unixodbc)" + homepage "http://www.iodbc.org/dataspace/iodbc/wiki/iODBC/" + url "https://github.com/openlink/iODBC/archive/v3.52.12.tar.gz" + sha256 "b2cab4f8fbd5388344d7ee0deb1a32310795b1e631c3505a5b515d90186e2516" + + depends_on "autoconf" => :build + depends_on "automake" => :build + depends_on "libtool" => :build + + conflicts_with "unixodbc", :because => "both install 'odbcinst.h' header" + + def install + system "./autogen.sh" + system "./configure", "--prefix=#{prefix}" + system "make", "install" + end + + test do + system bin/"iodbc-config", "--version" + end +end + diff --git a/coin/provisioning/common/macos/libiodbc.sh b/coin/provisioning/common/macos/libiodbc.sh index 79fa86c2..4ba0bbc3 100755 --- a/coin/provisioning/common/macos/libiodbc.sh +++ b/coin/provisioning/common/macos/libiodbc.sh @@ -41,7 +41,7 @@ set -ex source "${BASH_SOURCE%/*}/../unix/SetEnvVar.sh" brew update -brew install libiodbc +brew install ${BASH_SOURCE%/*}/libiodbc.rb # CPLUS_INCLUDE_PATH is set so clang and configure can find libiodbc -- cgit v1.2.3 From a3136bdbfef242339c002630f51b4bc0f5c05503 Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Tue, 3 Sep 2019 12:46:29 +0300 Subject: Provisioning: Disable or delete unneeded tasks and services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows have lot's of tasks and services running which are slowing the performance of build machine. Also 'windows' and 'windows defender' updates need's to be "permanently" disabled. Task-number: QTQAINFRA-3182 Change-Id: I8f6b237fcdc6e38fa9a06836d349821d86c58afc Reviewed-by: Tony Sarajärvi --- .../common/windows/disable-schedule-tasks.ps1 | 88 ++++++++++++++++++++++ .../common/windows/disable-update-orchestrator.ps1 | 16 ++++ .../common/windows/disable-windefender.ps1 | 51 +++++++++++++ .../windows/disable-windows-file-protector.ps1 | 4 + .../windows/disable-windows-module-installer.ps1 | 4 + .../windows/disable-windows-update-medic.ps1 | 25 ++++++ coin/provisioning/common/windows/helpers.ps1 | 20 +++++ 7 files changed, 208 insertions(+) create mode 100644 coin/provisioning/common/windows/disable-schedule-tasks.ps1 create mode 100644 coin/provisioning/common/windows/disable-update-orchestrator.ps1 create mode 100644 coin/provisioning/common/windows/disable-windows-file-protector.ps1 create mode 100644 coin/provisioning/common/windows/disable-windows-module-installer.ps1 create mode 100644 coin/provisioning/common/windows/disable-windows-update-medic.ps1 (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/disable-schedule-tasks.ps1 b/coin/provisioning/common/windows/disable-schedule-tasks.ps1 new file mode 100644 index 00000000..351f10c6 --- /dev/null +++ b/coin/provisioning/common/windows/disable-schedule-tasks.ps1 @@ -0,0 +1,88 @@ +. "$PSScriptRoot\helpers.ps1" + +# This script will remove unneeded Tasks from Task Scheduler + +# Application Experience 'Microsoft Compatibility Appraiser' - "Collects program telemetry information if opted-in to the Microsoft Customer Experience Improvement Program." +DisableSchedulerTask "Application Experience\Microsoft Compatibility Appraiser" + +# Application Experience 'ProgramDataUpdater' - "Collects program telemetry information if opted-in to the Microsoft Customer Experience Improvement Program" +DisableSchedulerTask "Application Experience\ProgramDataUpdater" + +# Autochk 'Proxy' - "This task collects and uploads autochk SQM data if opted-in to the Microsoft Customer Experience Improvement Program." +DisableSchedulerTask "Autochk\Proxy" + +# Chkdsk 'ProactiveScan' - "NTFS Volume Health Scan" +DisableSchedulerTask "Chkdsk\ProactiveScan" + +# Chkdsk 'SyspartRepair' +DeleteSchedulerTask "Chkdsk\SyspartRepair" + +# Customer Experience Improvement Program 'Consolidator' - "If the user has consented to participate in the Windows Customer Experience Improvement Program, this job collects and sends usage data to Microsoft." +DisableSchedulerTask "Customer Experience Improvement Program\Consolidator" + +# Customer Experience Improvement Program 'sbCeip' - "The USB CEIP (Customer Experience Improvement Program) task collects Universal Serial Bus related statistics and information about your machine and sends it to the Windows Device Connectivity engineering group at Microsoft. The information received is used to help improve the reliability, stability, and overall functionality of USB in Windows. If the user has not consented to participate in Windows CEIP, this task does not do anything." +DisableSchedulerTask "Customer Experience Improvement Program\UsbCeip" + +# Device Information 'Device' +DisableSchedulerTask "Device Information\Device" + +# Diagnosis 'Scheduled' - "The Windows Scheduled Maintenance Task performs periodic maintenance of the computer system by fixing problems automatically or reporting them through Security and Maintenance." +DisableSchedulerTask "Diagnosis\Scheduled" + +# DiskDiagnostic 'Microsoft-Windows-DiskDiagnosticDataCollector' - "The Windows Disk Diagnostic reports general disk and system information to Microsoft for users participating in the Customer Experience Program." +DisableSchedulerTask "DiskDiagnostic\Microsoft-Windows-DiskDiagnosticDataCollector" + +# ExploitGuard 'ExploitGuard MDM policy Refresh' - "Task for applying changes to the machine's Exploit Protection settings." +DisableSchedulerTask "ExploitGuard\ExploitGuard MDM policy Refresh" + +# Feedback/Siuf 'DmClient' +DisableSchedulerTask "Feedback\Siuf\DmClient" + +# Feedback/Siuf 'DmClient'OnScenarioDownload' +DisableSchedulerTask "Feedback\Siuf\DmClientOnScenarioDownload" + +# File Classification Infrastructure 'Property Definition Sync' +DisableSchedulerTask "File Classification Infrastructure\Property Definition Sync" + +# InstallService 'ScanForUpdates' +DisableSchedulerTask "InstallService\ScanForUpdates" + +# InstallService 'ScanForUpdatesAsUser' +DisableSchedulerTask "InstallService\ScanForUpdatesAsUser" + +# LanguageComponentsInstaller 'Installation' - "Install language components that match the user's language list." +DisableSchedulerTask "LanguageComponentsInstaller\Installation" + +# LanguageComponentsInstaller 'ReconcileLanguageResources' - "Install language components that match the user's language list." +DisableSchedulerTask "LanguageComponentsInstaller\ReconcileLanguageResources" + +# PI 'Secure-Boot-Update' - "This task updates the Secure Boot variables." +DisableSchedulerTask "PI\Secure-Boot-Update" + +# PI 'Sqm-Tasks' - "This task gathers information about the Trusted Platform Module (TPM), Secure Boot, and Measured Boot." +DisableSchedulerTask "PI\Sqm-Tasks" + +# Power Efficiency Diagnotics 'AnalyzeSystem' - "This task analyzes the system looking for conditions that may cause high energy use." +DisableSchedulerTask "PushToInstall\Registration" + +# Servicing 'StartComponentCleanup' +DisableSchedulerTask "Servicing\StartComponentCleanup" + +# SettingSync 'BackgroundUploadTask' +DeleteSchedulerTask "SettingSync\BackgroundUploadTask" + +# SoftwareProtectionPlatform 'SvcRestartTask' - "This task restarts the Software Protection Platform service at the specified time" +DeleteSchedulerTask "SoftwareProtectionPlatform\SvcRestartTask" + +# SoftwareProtectionPlatform 'SvcRestartTaskLogon' - "This task restarts the Software Protection Platform service at the specified time" +DisableSchedulerTask "SoftwareProtectionPlatform\SvcRestartTaskLogon" + +# SoftwareProtectionPlatform 'SvcRestartTaskNetwork' - "This task restarts the Software Protection Platform service when a new network is detected" +DisableSchedulerTask "SoftwareProtectionPlatform\SvcRestartTaskNetwork" + +# UNP 'RunUpdateNotificationMgr' +DeleteSchedulerTask "UNP\RunUpdateNotificationMgr" + + + + diff --git a/coin/provisioning/common/windows/disable-update-orchestrator.ps1 b/coin/provisioning/common/windows/disable-update-orchestrator.ps1 new file mode 100644 index 00000000..5ae8fb0d --- /dev/null +++ b/coin/provisioning/common/windows/disable-update-orchestrator.ps1 @@ -0,0 +1,16 @@ +# Disable UpdateOrchestrator + +$name = "UpdateOrchestrator" +$path = "C:\Windows\System32\Tasks\Microsoft\Windows\$name" + +takeown /F $path /A /R +icacls $path /grant Administrators:F /T +SCHTASKS /Change /TN "Microsoft\Windows\$name\Reboot" /DISABLE +del "$path\Schedule Scan" +del "$path\Schedule Scan Static Task" +del "$path\Backup Scan" +del "$path\UpdateModelTask" +del "$path\USO_UxBroker" + +# Disable Update orchestrator service +reg.exe ADD "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\UsoSvc" /V Start /T REG_dWORD /D 4 /F diff --git a/coin/provisioning/common/windows/disable-windefender.ps1 b/coin/provisioning/common/windows/disable-windefender.ps1 index e508641e..02ed54da 100644 --- a/coin/provisioning/common/windows/disable-windefender.ps1 +++ b/coin/provisioning/common/windows/disable-windefender.ps1 @@ -1,3 +1,36 @@ +########################################################################### +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + # Turning off win defender. # # If disabled manually, windows will automatically enable it after @@ -6,3 +39,21 @@ . "$PSScriptRoot\helpers.ps1" Run-Executable "reg.exe" "ADD `"HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows Defender`" /V DisableAntiSpyware /T REG_dWORD /D 1 /F" + +# 'Windows Defender Cache Maintenance' - "Periodic maintenance task." +DisableSchedulerTask "Windows Defender\Windows Defender Cache Maintenance" + +# 'Windows Defender Cleanup' - "Periodic cleanup task." +DisableSchedulerTask "Windows Defender\Windows Defender Cleanup" + +# 'Windows Defender Scheduled Scan' - "Periodic scan task." +DisableSchedulerTask "Windows Defender\Windows Defender Scheduled Scan" + +# 'Windows Defender Verification' - "Periodic verification task." +DisableSchedulerTask "Windows Defender\Windows Defender Verification" + +# Disable 'QueueReporting' - "Windows Error Reporting task to process queued reports." +DisableSchedulerTask "Windows Error Reporting\QueueReporting" + +# Disable WindowsUpdate from Task Scheduler +DisableSchedulerTask "WindowsUpdate\Scheduled Start" diff --git a/coin/provisioning/common/windows/disable-windows-file-protector.ps1 b/coin/provisioning/common/windows/disable-windows-file-protector.ps1 new file mode 100644 index 00000000..f1d09fb5 --- /dev/null +++ b/coin/provisioning/common/windows/disable-windows-file-protector.ps1 @@ -0,0 +1,4 @@ +# Disable Windows File Protection +# Windows File Protection feature in Microsoft Windows prevents programs from replacing critical Windows system files. + +reg.exe ADD "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon" /V SFCDisable /T REG_dWORD /D 0xffffff9d /F diff --git a/coin/provisioning/common/windows/disable-windows-module-installer.ps1 b/coin/provisioning/common/windows/disable-windows-module-installer.ps1 new file mode 100644 index 00000000..c7f85ff7 --- /dev/null +++ b/coin/provisioning/common/windows/disable-windows-module-installer.ps1 @@ -0,0 +1,4 @@ +# Disable Windows Module Installer (Trusted Installer). +# Trusted Installe enables installation, modification, and removal of Windows updates and optional components. +# If this service is disabled, install or uninstall of Windows updates might fail for this computer. +sc.exe config TrustedInstaller start=disabled diff --git a/coin/provisioning/common/windows/disable-windows-update-medic.ps1 b/coin/provisioning/common/windows/disable-windows-update-medic.ps1 new file mode 100644 index 00000000..6accbd3b --- /dev/null +++ b/coin/provisioning/common/windows/disable-windows-update-medic.ps1 @@ -0,0 +1,25 @@ +# Windows Update Medic Service (WaaSMedicSvc)'PerformRemediation' helps recover update-related services to the supported configuration. +# WaasMedicSvc keeps re-starting Windows Update, even if it disabled manually. +# Even Admin user don't have privileged to disable PerformRemediation from Task Scheduler which means that WaaSMedicSvc.dll need's to be removed from the system + +$limit = (Get-Date).AddMinutes(20) +$path = "C:\Windows\System32\WaaSMedicSvc.dll" + +DO { + takeown /F $path + icacls $path /grant Administrators:f + Write-host "Deleting $path" + + Try { + del $path + } + Catch [System.UnauthorizedAccessException] { + Write-host "Access to the path '$path' is denied." + Continue + } + + if ((Get-Date) -gt $limit) { + exit 1 + } + +}while (Test-Path -Path "$path") diff --git a/coin/provisioning/common/windows/helpers.ps1 b/coin/provisioning/common/windows/helpers.ps1 index ba35f6b5..0f39beed 100644 --- a/coin/provisioning/common/windows/helpers.ps1 +++ b/coin/provisioning/common/windows/helpers.ps1 @@ -222,3 +222,23 @@ function Remove { } } } + +function DisableSchedulerTask { + + Param ( + [string]$Task = $(BadParam("a task")) + ) + + Write-Host "Disabling $Task from Task Scheduler" + SCHTASKS /Change /TN "Microsoft\Windows\$Task" /DISABLE +} + +function DeleteSchedulerTask { + + Param ( + [string]$Task = $(BadParam("a task")) + ) + + Write-Host "Disabling $Task from Task Scheduler" + SCHTASKS /DELETE /TN "Microsoft\Windows\$Task" /F +} -- cgit v1.2.3 From 0cf3fdabe315d62066a7642787a5058a737a79e4 Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Thu, 1 Aug 2019 14:51:40 +0300 Subject: Provisioning: Upgrade Python pip version for Windows machines Task-number: QTQAINFRA-2564 Change-Id: I10402bb3169b483c71e44db7ecba443eb4f46dc1 Reviewed-by: Liang Qi --- coin/provisioning/common/windows/python.ps1 | 5 ++++- coin/provisioning/common/windows/python3.ps1 | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/python.ps1 b/coin/provisioning/common/windows/python.ps1 index aee4c199..342358d5 100644 --- a/coin/provisioning/common/windows/python.ps1 +++ b/coin/provisioning/common/windows/python.ps1 @@ -1,6 +1,6 @@ ############################################################################# ## -## Copyright (C) 2016 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -75,6 +75,9 @@ if (($archVer -eq 32) -And (Is64BitWinHost)) { Run-Executable "$targetDir\python.exe" "-m ensurepip" +Write-Host "Upgrade pip to the latest version available." +Run-Executable "$targetDir\python.exe" "-m pip install --upgrade pip" + # Install python virtual env if (IsProxyEnabled) { $proxy = Get-Proxy diff --git a/coin/provisioning/common/windows/python3.ps1 b/coin/provisioning/common/windows/python3.ps1 index c05c022c..8fbd615c 100644 --- a/coin/provisioning/common/windows/python3.ps1 +++ b/coin/provisioning/common/windows/python3.ps1 @@ -1,6 +1,6 @@ ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Copyright (C) 2017 Pelagicore AG ## Contact: http://www.qt.io/licensing/ ## @@ -79,6 +79,10 @@ if (IsProxyEnabled) { Write-Host "Using proxy ($proxy) with pip" $pip_args = "--proxy=$proxy" } + +Write-Host "Upgrade pip3 to the latest version available." +Run-Executable "$install_path\python.exe" "-m pip install --upgrade pip" + Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv wheel" # Install all needed packages in a special wheel cache directory -- cgit v1.2.3 From e6fccbf2407bcb7f4006fd70a2af16451e3a44f1 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 31 Jul 2019 12:05:34 +0200 Subject: Remove the progress bar from the sdkmanager output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleans up many thousands of useless lines from the logs. Change-Id: I2673f65b713996637d2161da4db726823e17c0df Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/linux/android_linux.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/android_linux.sh b/coin/provisioning/common/linux/android_linux.sh index 5840d4b4..b83c1c6e 100755 --- a/coin/provisioning/common/linux/android_linux.sh +++ b/coin/provisioning/common/linux/android_linux.sh @@ -80,14 +80,22 @@ else sudo chown -R qt:users "$targetFolder" fi +# Run the following command under `eval` or `sh -c` so that the shell properly splits it +sdkmanager_no_progress_bar_cmd="tr '\r' '\n' | grep -v '^\[[ =]*\]'" + echo "Running SDK manager for platforms;$sdkApiLevel, platform-tools and build-tools;$sdkBuildToolsVersion." # shellcheck disable=SC2031 if [ "$http_proxy" != "" ]; then proxy_host=$(echo "$proxy" | cut -d'/' -f3 | cut -d':' -f1) proxy_port=$(echo "$proxy" | cut -d':' -f3) - echo "y" |"$sdkTargetFolder/tools/bin/sdkmanager" --no_https --proxy=http --proxy_host="$proxy_host" --proxy_port="$proxy_port" "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" + echo "y" | "$sdkTargetFolder/tools/bin/sdkmanager" \ + --no_https --proxy=http --proxy_host="$proxy_host" --proxy_port="$proxy_port" \ + "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" \ + | eval $sdkmanager_no_progress_bar_cmd else - echo "y" |"$sdkTargetFolder/tools/bin/sdkmanager" "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" + echo "y" | "$sdkTargetFolder/tools/bin/sdkmanager" \ + "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" \ + | eval $sdkmanager_no_progress_bar_cmd fi echo "Checking the contents of Android SDK..." @@ -106,8 +114,11 @@ echo "Android SDK API level = $sdkApiLevel" >> ~/versions.txt echo "Android NDK = $ndkVersion" >> ~/versions.txt cd "$sdkTargetFolder/tools/bin" -./sdkmanager --install "emulator" -echo "y" | ./sdkmanager --install "system-images;android-21;google_apis;x86" +./sdkmanager --install "emulator" \ + | eval $sdkmanager_no_progress_bar_cmd +echo "y" | ./sdkmanager --install "system-images;android-21;google_apis;x86" \ + | eval $sdkmanager_no_progress_bar_cmd + echo "Checking the contents of Android SDK again..." ls -l "$sdkTargetFolder" -- cgit v1.2.3 From bdc5dae67b8d36d797a42c5e2dc72a1510b81e27 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Mon, 12 Aug 2019 19:14:10 +0200 Subject: Speed up provisioning on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I6e4e5621ed0b063215476ef51d61ef1f38198d0c Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/unix/install_protobuff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/unix/install_protobuff.sh b/coin/provisioning/common/unix/install_protobuff.sh index 59e9b699..a9b2cfd0 100755 --- a/coin/provisioning/common/unix/install_protobuff.sh +++ b/coin/provisioning/common/unix/install_protobuff.sh @@ -64,7 +64,7 @@ if uname -a |grep -q Darwin; then else ./configure fi -make +make -j5 sudo make install # Refresh shared library cache if OS isn't macOS -- cgit v1.2.3 From b97df21bef1d6a9deab1c62eaa1cbb8f477819ad Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 14 Aug 2019 15:24:32 +0200 Subject: Brew update only once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I6c2650574b6b62ceedfc95a9caab7ca26a6fa10b Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/macos/homebrew.sh | 2 ++ coin/provisioning/common/macos/libiodbc.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/macos/homebrew.sh b/coin/provisioning/common/macos/homebrew.sh index a6efe123..f7042de3 100755 --- a/coin/provisioning/common/macos/homebrew.sh +++ b/coin/provisioning/common/macos/homebrew.sh @@ -1,3 +1,5 @@ #!/usr/bin/env bash # Will install homebrew package manager for macOS /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" Date: Thu, 15 Aug 2019 13:55:07 +0200 Subject: Increase open files and processes limit on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: If93bbe067f76168a7c131c6ced187f00a8d908e1 Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/macos/increase_limits.sh | 72 ++++++++++++++++++++++ .../provisioning/common/macos/limit.maxfiles.plist | 20 ++++++ coin/provisioning/common/macos/limit.maxproc.plist | 20 ++++++ 3 files changed, 112 insertions(+) create mode 100755 coin/provisioning/common/macos/increase_limits.sh create mode 100644 coin/provisioning/common/macos/limit.maxfiles.plist create mode 100644 coin/provisioning/common/macos/limit.maxproc.plist (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/macos/increase_limits.sh b/coin/provisioning/common/macos/increase_limits.sh new file mode 100755 index 00000000..666d9067 --- /dev/null +++ b/coin/provisioning/common/macos/increase_limits.sh @@ -0,0 +1,72 @@ +#!/bin/sh + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +# Increase the soft and hard RLIMIT_NOFILE and RLIMIT_NPROC limits. +# By default they are 256/unlimited and 709/1064 +# and they sometimes create problems to the build process and telegraf. + + +# Must have this variable set in the environment +[ x"$PROVISION_DIR" = x ] && exit 38 + + +echo "Current limits are:" +ulimit -a +launchctl limit + +sudo cp $PROVISION_DIR/common/macos/limit.maxfiles.plist /Library/LaunchDaemons/ +sudo cp $PROVISION_DIR/common/macos/limit.maxproc.plist /Library/LaunchDaemons/ + +sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist +sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist + +sudo chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist +sudo chmod 644 /Library/LaunchDaemons/limit.maxproc.plist + +# Activate the new limits immediately (not for the current session though) +sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist +sudo launchctl load -w /Library/LaunchDaemons/limit.maxproc.plist + +echo "After adjusting, limits are:" +ulimit -a +launchctl limit + + +# NOTE: If the limits are not increased enough, it might be because of +# restrictions set by the kernel. They can be temporarily altered with: + +# sudo sysctl -w kern.maxproc=xxxx +# sudo sysctl -w kern.maxprocperuid=xxx diff --git a/coin/provisioning/common/macos/limit.maxfiles.plist b/coin/provisioning/common/macos/limit.maxfiles.plist new file mode 100644 index 00000000..1bf85214 --- /dev/null +++ b/coin/provisioning/common/macos/limit.maxfiles.plist @@ -0,0 +1,20 @@ + + + + + Label + limit.maxfiles + ProgramArguments + + launchctl + limit + maxfiles + 65536 + 524288 + + RunAtLoad + + ServiceIPC + + + diff --git a/coin/provisioning/common/macos/limit.maxproc.plist b/coin/provisioning/common/macos/limit.maxproc.plist new file mode 100644 index 00000000..0c2cc0e9 --- /dev/null +++ b/coin/provisioning/common/macos/limit.maxproc.plist @@ -0,0 +1,20 @@ + + + + + Label + limit.maxproc + ProgramArguments + + launchctl + limit + maxproc + 8192 + 32768 + + RunAtLoad + + ServiceIPC + + + -- cgit v1.2.3 From c7f87acb3717b3b2231e9ded71e481e82a742596 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Mon, 9 Sep 2019 10:45:47 +0200 Subject: Do not silence standard error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provisioning was failing and the logs did not provide any useful information because of the silenced squish install command. So I went through all the scripts and I re-enabled stderr logging where I found it silenced (with the exception of where it was purposeful or harmless). Change-Id: I5fa0b0ba4362970b214c150d399b6720b1780e1e Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/linux/disable-ntp_linux.sh | 2 +- coin/provisioning/common/unix/squishInstall.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/disable-ntp_linux.sh b/coin/provisioning/common/linux/disable-ntp_linux.sh index c509d36b..dc592553 100755 --- a/coin/provisioning/common/linux/disable-ntp_linux.sh +++ b/coin/provisioning/common/linux/disable-ntp_linux.sh @@ -44,5 +44,5 @@ elif cat /etc/os-release | grep "PRETTY_NAME" | grep -q "Leap 15"; then elif cat /etc/os-release |grep "SUSE Linux Enterprise Server 15"; then sudo timedatectl set-ntp false else - (systemctl &>/dev/null && sudo systemctl disable ntpd) || sudo /sbin/chkconfig ntpd off + sudo systemctl disable ntpd || sudo /sbin/chkconfig ntpd off fi diff --git a/coin/provisioning/common/unix/squishInstall.sh b/coin/provisioning/common/unix/squishInstall.sh index b40c1922..f1b5e6b5 100755 --- a/coin/provisioning/common/unix/squishInstall.sh +++ b/coin/provisioning/common/unix/squishInstall.sh @@ -122,7 +122,7 @@ function MountAndInstall { sudo cp $targetFileMount /tmp UnMount sudo hdiutil attach "/tmp/$targetFile" - sudo /Volumes/froglogic\ Squish/Install\ Squish.app/Contents/MacOS/Squish unattended=1 targetdir="$targetDirectory/package" qtpath="$targetDirectory" > /dev/null 2>&1 + sudo /Volumes/froglogic\ Squish/Install\ Squish.app/Contents/MacOS/Squish unattended=1 targetdir="$targetDirectory/package" qtpath="$targetDirectory" > /dev/null mountFolder="/Volumes/froglogic Squish" UnMount elif [[ $targetFile == *.run ]]; then @@ -130,7 +130,7 @@ function MountAndInstall { sudo cp $targetFileMount $targetDirectory UnMount sudo chmod +x $targetDirectory/$targetFile - sudo $targetDirectory/$targetFile unattended=1 targetdir="$targetDirectory/package" qtpath="$targetDirectory" > /dev/null 2>&1 + sudo $targetDirectory/$targetFile unattended=1 targetdir="$targetDirectory/package" qtpath="$targetDirectory" > /dev/null sudo rm -fr "$targetDirectory/$targetFile" if uname -a |grep -q "Ubuntu"; then sudo mkdir /usr/lib/tcl8.6 -- cgit v1.2.3 From c0fce7a3b37db53289ffd1e44c58f0bfc0601229 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 11 Sep 2019 19:24:25 +0200 Subject: Refactor DownloadURL.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-detects and verifies all kinds of hashes, not only SHA1. Hash verification is now an autonomous function. If the file is found at the destination, skip redownloading it if the hash is OK. Runs on POSIX sh, not requiring bash. Retain compatibility with previous version. Change-Id: I9930336030aa776f49dc460fe766f3e831700255 Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/unix/DownloadURL.sh | 84 ++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 23 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/unix/DownloadURL.sh b/coin/provisioning/common/unix/DownloadURL.sh index 996c99da..da6d772f 100755 --- a/coin/provisioning/common/unix/DownloadURL.sh +++ b/coin/provisioning/common/unix/DownloadURL.sh @@ -1,8 +1,8 @@ -#!/usr/bin/env bash +#!/bin/sh ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -33,31 +33,69 @@ ## ############################################################################# + # A helper script used for downloading a file from a URL or an alternative -# URL. Also the SHA1 is checked for the file. Target filename should also -# be given. -# -# If called directly from another script, it will exit the parent script -# as well, if not called in its own subshell with parentheses. +# URL. Also the SHA is checked for the file (SHA algorithm is autodetected +# based on the SHA length). Target filename should also be given. + +############################ BOILERPLATE ############################### +command -v sha1sum >/dev/null || alias sha1sum='shasum -a 1' +command -v sha256sum >/dev/null || alias sha256sum='shasum -a 256' +command -v sha384sum >/dev/null || alias sha384sum='shasum -a 384' +command -v sha512sum >/dev/null || alias sha512sum='shasum -a 512' +######################################################################## + + +VerifyHash () { + file=$1 + expectedHash=$2 + + if [ ! -f "$file" ] + then return 2 # file does not exist + fi -set -ex -function DownloadURL { + hashLength="$(echo "$expectedHash" | wc -c | sed 's/ *//g')" + # Use backticks because of bug with bash-3 (default on macOS), + # caused when there are unbalanced parentheses inside $() + # shellcheck disable=SC2006 + hash=`case "$hashLength" in + 41) sha1sum "$file" ;; + 65) sha256sum "$file" ;; + 97) sha384sum "$file" ;; + 129) sha512sum "$file" ;; + *) echo "FATAL! Unknown hash length: $hashLength" 1>&2 && exit 1 ;; + esac | cut -d ' ' -f 1` + + if [ ! "$expectedHash" = "$hash" ] + then + echo "FAIL! wrong file hash: $file $hash" 1>&2 + return 1 + fi + echo "OK verified integrity of: $file" +} + +# Check if file already exists and is good, otherwise download it +DownloadURL () { url=$1 - url_alt=$2 - expectedSha1=$3 + url2=$2 + expectedHash=$3 targetFile=$4 - echo "Downloading from primary URL '$url'" - curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" || { - echo "Failed to download '$url' multiple times" - echo "Downloading from alternative URL '$url_alt'" - curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url_alt" || { echo 'Failed to download even from alternative url'; exit 1; } - } - - echo "Checking SHA1 on PKG '$targetFile'" - echo "$expectedSha1 *$targetFile" > "$targetFile.sha1" - sha1sum --check "$targetFile.sha1" - rm -f "$targetFile.sha1" + if VerifyHash "$targetFile" "$expectedHash" + then + echo "Skipping download, found and validated existing file: $targetFile" + else + echo "Downloading from primary URL: $url" + if ! curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" + then + echo "FAIL! to download, trying alternative URL: $url2" 1>&2 + if ! curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url2" + then + echo 'FAIL! to download even from alternative url' 1>&2 + return 1 + fi + fi + VerifyHash "$targetFile" "$expectedHash" + fi } - -- cgit v1.2.3 From fbe3a8d84e943b8e1ff89389bc6af9e09532722a Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 24 Jul 2019 16:17:46 +0200 Subject: Install telegraf on all provisioned VMs Install binaries as early as possible in the provisioning process, and start them, in order to monitor provisioning too. To achieve this, some OS auto-detection logic is introduced in shell script common.sourced.sh. The script can be sourced and used in all kind of other generic scripts. ioping is also installed from custom-compiled binaries in order to monitor the disk latency of the VMs. Reason we don't use ioping from the repositories, is that the units differ between ioping versions (new ioping reports latencies in nanoseconds, old in microseconds). Fixes: QTQAINFRA-3092 Change-Id: I9d9afb791955725d4bd0b32dae97dfc0bd4a76a3 Reviewed-by: Heikki Halmet --- .../linux/devel_languages_go.SUSE_repo_key.pub | 19 +++ coin/provisioning/common/linux/telegraf-coin.conf | 47 +++++++ coin/provisioning/common/macos/telegraf-coin.conf | 51 ++++++++ .../common/shared/telegraf/influxdb.repo_key.pub | 52 ++++++++ .../common/shared/telegraf/telegraf-ioping.sh | 26 ++++ .../shared/telegraf/telegraf_packages.sha256.txt | 6 + coin/provisioning/common/unix/common.sourced.sh | 139 +++++++++++++++++++++ coin/provisioning/common/unix/telegraf_install.sh | 98 +++++++++++++++ coin/provisioning/common/unix/telegraf_password.sh | 60 +++++++++ coin/provisioning/common/windows/helpers.ps1 | 2 +- .../common/windows/install_telegraf.ps1 | 68 ++++++++++ .../provisioning/common/windows/telegraf-coin.conf | 41 ++++++ .../common/windows/telegraf_password.ps1 | 39 ++++++ 13 files changed, 647 insertions(+), 1 deletion(-) create mode 100644 coin/provisioning/common/linux/devel_languages_go.SUSE_repo_key.pub create mode 100644 coin/provisioning/common/linux/telegraf-coin.conf create mode 100644 coin/provisioning/common/macos/telegraf-coin.conf create mode 100644 coin/provisioning/common/shared/telegraf/influxdb.repo_key.pub create mode 100644 coin/provisioning/common/shared/telegraf/telegraf-ioping.sh create mode 100644 coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt create mode 100644 coin/provisioning/common/unix/common.sourced.sh create mode 100755 coin/provisioning/common/unix/telegraf_install.sh create mode 100755 coin/provisioning/common/unix/telegraf_password.sh create mode 100644 coin/provisioning/common/windows/install_telegraf.ps1 create mode 100644 coin/provisioning/common/windows/telegraf-coin.conf create mode 100644 coin/provisioning/common/windows/telegraf_password.ps1 (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/devel_languages_go.SUSE_repo_key.pub b/coin/provisioning/common/linux/devel_languages_go.SUSE_repo_key.pub new file mode 100644 index 00000000..48162c56 --- /dev/null +++ b/coin/provisioning/common/linux/devel_languages_go.SUSE_repo_key.pub @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.5 (GNU/Linux) + +mQGiBEyvJq8RBAD64FtKbU3VFsZ4PAzcSi9rETc1qXZLVQYtrE38niriIJUuT/tC +kxGjkduD7ozP6rxL+M8fk9UdbbmzhHhXzQT2blyYy2iwpZP5+AFeQawz+oyMRVze +x6IY5qO31TiRu+aEBptk6nco4DULlKABlvtjKI+gJ4buJm+TumyrPaxe9wCgpFwl +O0fEjpOfQd+6Du0sCnPX7+8D/3xYVj9q5ILmrJ9zehBKjRYEcaP0rZVqudH51FUS +I0LQNHsJdzKLzhT+Zj7TJU4KnCN4sX7amfYL3GT9qyZEBDx2alubvEsI36wtNq0i +ewU4XmyZ+Rd7Q4mAPNoBGe5DsUA8wn/7iHGg9aJHu+GXvH7T3DVoUxW5W2KqPw+h +jzqGBACWGuuCefa683uf6aKqpTS3JB+01BDPf+sG9qJ+k7xcJOsEiXIe1X5vH9ov +kEvTM12kzY7SGwlsFsTDT1xioDl62g5nPlZMXErUBVnjoMtjl9bDwUPJY/vP1S9M +VaONBMoJEON3HRui9jhdEK25LTdrcFlFpRzJI3AgdXwy8aQyiLRGZGV2ZWw6bGFu +Z3VhZ2VzOmdvIE9CUyBQcm9qZWN0IDxkZXZlbDpsYW5ndWFnZXM6Z29AYnVpbGQu +b3BlbnN1c2Uub3JnPohmBBMRAgAmBQJc9DoCAhsDBQkUY8NTBgsJCAcDAgQVAggD +BBYCAwECHgECF4AACgkQr0z+TTB9e/lQZgCfXIjj30HyCPvUbSbSd6zOS0seAegA +n3DXayMpRphNmyJgLvl5qKfsX7f6iEYEExECAAYFAkyvJq8ACgkQOzARt2udZSOg +qQCeOba7kjk7CI+64GOkhhTGSdjnjdQAn2IC5tx+PW38XKwl3Jr5YVp0RyR0 +=AOj6 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/coin/provisioning/common/linux/telegraf-coin.conf b/coin/provisioning/common/linux/telegraf-coin.conf new file mode 100644 index 00000000..33a9c6de --- /dev/null +++ b/coin/provisioning/common/linux/telegraf-coin.conf @@ -0,0 +1,47 @@ +[global_tags] +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 100 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "5s" + precision = "" + logfile = "/var/log/telegraf-coin.log" + hostname = "$COIN_UNIQUE_JOB_ID" + omit_hostname = false +[[outputs.influxdb]] + urls= ["https://qtinfluxdb01.intra.qt.io:8086"] + insecure_skip_verify = true + database = "coin_vms" + skip_database_creation = true + username = "coin_vms_writer" + password = "COIN_VMS_WRITER_PASS" +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] +[[inputs.diskio]] +[[inputs.kernel]] +[[inputs.mem]] +[[inputs.processes]] +[[inputs.swap]] +[[inputs.system]] +[[inputs.kernel_vmstat]] # linux-only +[[inputs.netstat]] + + +# Monitor Coin process +[[inputs.procstat]] + exe = "coin-" + pid_finder = "native" + + +# Monitor I/O latency +[[inputs.exec]] + data_format = "influx" + commands = [ "/usr/bin/telegraf-ioping.sh /" ] diff --git a/coin/provisioning/common/macos/telegraf-coin.conf b/coin/provisioning/common/macos/telegraf-coin.conf new file mode 100644 index 00000000..24440a05 --- /dev/null +++ b/coin/provisioning/common/macos/telegraf-coin.conf @@ -0,0 +1,51 @@ +[global_tags] +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 100 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "5s" + precision = "" + logfile = "/var/log/telegraf-coin.log" + hostname = "$COIN_UNIQUE_JOB_ID" + omit_hostname = false +[[outputs.influxdb]] + urls= ["https://qtinfluxdb01.intra.qt.io:8086"] + insecure_skip_verify = true + database = "coin_vms" + skip_database_creation = true + username = "coin_vms_writer" + password = "COIN_VMS_WRITER_PASS" +#[[inputs.cpu]] # linux-only +# percpu = true +# totalcpu = true +# collect_cpu_time = false +# report_active = false +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] +#[[inputs.diskio]] # linux-only +[[inputs.kernel]] +[[inputs.mem]] +[[inputs.processes]] +[[inputs.swap]] +[[inputs.system]] +#[[inputs.kernel_vmstat]] # linux-only +[[inputs.netstat]] + + +# Monitor Coin process +[[inputs.procstat]] + exe = "coin-" + # Buggy on macOS if using the "native" pid_finder, see: + # https://github.com/influxdata/telegraf/issues/6265 + # Tested 1.12.1 version that "native" pid_finder has been fixed, + # but it is slow and times out frequently + pid_finder = "pgrep" + + +# Monitor I/O latency +[[inputs.exec]] + data_format = "influx" + commands = [ "/usr/bin/telegraf-ioping.sh /" ] diff --git a/coin/provisioning/common/shared/telegraf/influxdb.repo_key.pub b/coin/provisioning/common/shared/telegraf/influxdb.repo_key.pub new file mode 100644 index 00000000..c97d5933 --- /dev/null +++ b/coin/provisioning/common/shared/telegraf/influxdb.repo_key.pub @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFYJmwQBEADCw7mob8Vzk+DmkYyiv0dTU/xgoSlp4SQwrTzat8MB8jxmx60l +QjmhqEyuB8ho4zzZF9KV+gJWrG6Rj4t69JMTJWM7jFz+0B1PC7kJfNM+VcBmkTnj +fP+KJjqz50ETnsF0kQTG++UJeRYjG1dDK0JQNQJAM6NQpIWJI339lcDf15vzrMnb +OgIlNxV6j1ZZqkle4fvScF1NQxYScRiL+sRgVx92SI4SyD/xZnVGD/szB+4OCzah ++0Q/MnNGV6TtN0RiCDZjIUYiHoeT9iQXEONKf7T62T4zUafO734HyqGvht93MLVU +GQAeuyx0ikGsULfOsJfBmb3XJS9u+16v7oPFt5WIbeyyNuhUu0ocK/PKt5sPYR4u +ouPq6Ls3RY3BGCH9DpokcYsdalo51NMrMdnYwdkeq9MEpsEKrKIN5ke7fk4weamJ +BiLI/bTcfM7Fy5r4ghdI9Ksw/ULXLm4GNabkIOSfT7UjTzcBDOvWfKRBLX4qvsx4 +YzA5kR+nX85u6I7W10aSqBiaLqk6vCj0QmBmCjlSeYqNQqSzH/6OoL6FZ7lP6AiG +F2NyGveJKjugoXlreLEhOYp20F81PNwlRBCAlMC2Q9mpcFu0dtAriVoG4gVDdYn5 +t+BiGfD2rJlCinYLgYBDpTPcdRT3VKHWqL9fcC4HKmic0mwWg9homx550wARAQAB +tDFJbmZsdXhEQiBQYWNrYWdpbmcgU2VydmljZSA8c3VwcG9ydEBpbmZsdXhkYi5j +b20+iQI3BBMBCgAhBQJWCZsEAhsDBQsJCAcDBRUKCQgLBRYDAgEAAh4BAheAAAoJ +EGhKFM8lguDF9XEQAK9rREnZt6ujh7GXfeNki35bkn39q8GYh0mouShFbFY9o0i3 +UJVChsxokJSRPgFh9GOhOPTupl3rzfdpD+IlWI2Myt6han2HOjZKNZ4RGNrYJ5UR +uxt4dKMWlMbpkzL56bhHlx97RoXKv2d2zRQfw9nyZb6t3lw2k2kKXsMxjGa0agM+ +2SropwYOXdtkz8UWaGd3LYxwEvW3AuhI8EEEHdLetQaYe9sANDvUEofgFbdsuICH +9QLmbYavk7wyGTPBKfPBbeyTxwW2rMUnFCNccMKLm1i5NpZYineBtQbX2cfx9Xsk +1JLOzEBmNal53H2ob0kjev6ufzOD3s8hLu4KMCivbIz4YT3fZyeExn0/0lUtsQ56 +5fCxE983+ygDzKsCnfdXqm3GgjaI90OkNr1y4gWbcd5hicVDv5fD3TD9f0GbpDVw +yDz8YmvNzxMILt5Glisr6aH7gLG/u8jxy0D8YcBiyv5kfY4vMI2yXHpGg1cn/sVu +ZB01sU09VVIM2BznnimyAayI430wquxkZCyMx//BqFM1qetIgk1wDZTlFd0n6qtA +fDmXAC4s5pM5rfM5V57WmPaIqnRIaESJ35tFUFlCHfkfl/N/ribGVDg1z2KDW08r +96oEiIIiV4GfXl+NprJqpNS3Cn+aCXtd7/TsDScDEgs4sMaR29Lsf26cuWk8uQIN +BFYJmwQBEADDPi3fmwn6iwkiDcH2E2V31cHlBw9OdJfxKVUdyAQEhTtqmG9P8XFZ +ERRQF155XLQPLvRlUlq7vEYSROn5J6BAnsjdjsH9LmFMOEV8CIRCRIDePG/Mez2d +nIK5yiU6GkS3IFaQg2T9/tOBKxm0ZJPfqTXbT4jFSfvYJ3oUqc+AyYxtb8gj1GRk +X283/86/bA3C98u7re1vPtiDRyM8r0+lhEc59Yx/EAOL+X2gZyTgyUoH+LLuOWQK +s1egI8y80R8NZfM1nMiQk2ywMsTFwQjSVimScvzqv5Nt8k8CvHUQ3a6R+6doXGNX +5RnUqn9Qvmh0JY5sNgFsoaGbuk2PJrVaGBRnfnjaDqAlZpDhwkWhcCcguNhRbRHp +N7/a0pQr70bAG9VikzLyGC17EU0sxney/hyNHkr4Uyy2OXHpuJvRjVKy/BwZ3fxA +AYX2oZIOxQB3/OulzO/DppaCVhRtp1bt+Z5f+fpisiVb5DvZcMdeyAoQ4+oOr7v3 +EasIs2XYcQ+kOE3Y2kdlHWBeuXzxgWgJZ1OOpwGMjR3Uy6IwhuSWtreJBA4er+Df +vgSPwKBsRLNLbPe3ftjArnC5GfMiGgikVdAUdN4OkEqvUbkRoAVGKTOMLUKm+ZkG +OskJOVYS+JAina0qkYEFF7haycMjf9olhqLmTIC+6X7Ox9R2plaOhQARAQABiQIf +BBgBCgAJBQJWCZsEAhsMAAoJEGhKFM8lguDF8ZIP/1q9Sdz8oMvf9AJXZ7AYxm77 +V+kJzJqi62nZLWJnrFXDZJpU+LkYlb3fstsZ1rvBhnrEPSmFxoj72CP0RtcyX7wJ +dA7K1Fl9LpJi5H8300cC7UyG94MUYbrXijbLTbnFTfNr1tGx4a1T/7Yyxx/wZGrT +H/X8cvNybkl33SxDdlQQ9kx3lFOwC41e3TkGsUWxn3TCfvDh8VdA6Py6JeSPFGOb +MEO2/q7oUgvjfV+ivN5ayZi9bWgeqm1sgtmTHHQ4RqwwKrAb5ynXpn1b9QrkevgT +b91uzMA22Prl4DuzKiaMYDcZOQ3vtf0eFBP0GOSSgUKS4bQ3dGgi1JmQ7VuAM4uj ++Ug5TnGoLwclTwLksc7v89C5MMPgm2vVXvCUDzyzQA7bIHFeX+Rziby4nymec4Nr +eeXYNBJWrEp8XR7UNWmEgroXRoN1x9/6esh5pnoUXGAIWuKzSLQM70/wWxS67+v2 +aC1GNb+pXXAzYeIIiyLWaZwCSr8sWMvshFT9REk2+lnb6sAeJswQtfTUWI00mVqZ +dvI3Wys2h0IyIejuwetTUvGhr9VgpqiLLfGzGlt/y2sg27wdHzSJbMh0VrVAK26/ +BlvEwWDCFT0ZJUMG9Lvre25DD0ycbougLsRYjzmGb/3k3UktS3XTCxyBa/k3TPw3 +vqIHrEqk446nGPDqJPS5 +=9iF7 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/coin/provisioning/common/shared/telegraf/telegraf-ioping.sh b/coin/provisioning/common/shared/telegraf/telegraf-ioping.sh new file mode 100644 index 00000000..ad95eadc --- /dev/null +++ b/coin/provisioning/common/shared/telegraf/telegraf-ioping.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Measure I/O latency once, return data in InfluxDB format +# +# Run one ioping command for read, and one for write. +# Each one sends 3 requests and reports the minimum time, in nanoseconds. +# (Because of limitations of ioping, we can't just send one request and get +# the number back in the batch format. Additionally, the number seems to be +# fluctuating quite a bit so taking the smallest number out of 3 requests is +# stabilising it a bit.) + + +set -e + +[ x"$1" = x ] && echo "$0 takes a path as a first argument" && exit 1 + +# Try to run in high priority to avoid slow-downs because of +# factors other than I/O. +renice -n -10 -p $$ >/dev/null 2>&1 || true + + +rlatency="$(ioping -B -k -c 3 -i 0.1 "$1" | cut -d " " -f 5)" +wlatency="$(ioping -B -k -c 3 -i 0.1 -W "$1" | cut -d " " -f 5)" + +printf "ioping,path=$1 read_latency_ns=%u,write_latency_ns=%u\n" \ + $rlatency $wlatency diff --git a/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt b/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt new file mode 100644 index 00000000..7d1cdada --- /dev/null +++ b/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt @@ -0,0 +1,6 @@ +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_windows_i386.zip d2dea67a28fd7a9e21395d14f0b7ce1b6413e85cc9ffd89cc7ff8f35e8af5837 +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_windows_amd64.zip 88d8c8146b561f43726e61688b42d0b5ab88fc22607b94df5a2c8245bbb27e29 +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_darwin_amd64.tar.gz 58a2b294c1b6408c396b5526719cf7cad557f7075fed10ff0b2a74141ea2d629 +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_linux_amd64.tar.gz 070a76800d1a5f33413b8948286f75caa55ebd924f6fb6996a753b9979799ef5 +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_linux_i386.tar.gz 964077b6846708f579a100d7ea6090e2d8350ace75ec735d539bc452bc551038 +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1-static_linux_amd64.tar.gz bb03814c99c037ce0758e3884660910f2f6fd78f3d49a8e1ef531693b3cfe25d diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh new file mode 100644 index 00000000..a52880b2 --- /dev/null +++ b/coin/provisioning/common/unix/common.sourced.sh @@ -0,0 +1,139 @@ +#!/bin/sh + + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +# Script to be sourced from everywhere you need a common environment. Defines: +export PROVISIONING_DIR +export PROVISIONING_OS +export PROVISIONING_OS_ID +export PROVISIONING_ARCH +export CMD_PKG_INSTALL +export CMD_PKG_LOCALINSTALL +export CMD_INSTALL + + + +if [ x"$IS_PROVISIONING_COMMON_SOURCED" != x ] +then + echo "common.sourced.sh has already been sourced, re-sourcing skipped" + return +fi + +# Do not export; you want children to re-source, because they only inherit the +# variables but not the functions +IS_PROVISIONING_COMMON_SOURCED=1 + + +fatal () { + echo "$1" 1>&2 + if [ x"$2" != x ] + then exit "$2" + else exit + fi +} + +is_script_executed () { + [ x"$(basename "$0")" = x"$1" ] +} + + +is_script_executed common.sourced.sh \ + && fatal "Script common.sourced.sh should always be sourced, not executed" + + +set_common_environment () { + # Unfortunately we can't find the provisioning directory from a sourced + # script in a portable way + # PROVISIONING_DIR="$(dirname "$0")/../../" + + [ x"$PROVISIONING_DIR" = x ] \ + && fatal "PROVISIONING_DIR variable needs to be set before calling set_common_environment" + + uname_s="$(uname -s)" + case "$uname_s" in + Linux) + PROVISIONING_OS=linux + . /etc/os-release + PROVISIONING_OS_ID="$ID" + case "$PROVISIONING_OS_ID" in + suse|sles|opensuse*) + CMD_PKG_INSTALL="sudo zypper -nq install" + CMD_PKG_LOCALINSTALL="sudo zypper --no-gpg-checks -nq install" + ;; + debian|ubuntu) + CMD_PKG_INSTALL="sudo apt -y install" + CMD_PKG_LOCALINSTALL="sudo apt -y install" + ;; + rhel|centos|fedora) + CMD_PKG_INSTALL="sudo yum -y install" + CMD_PKG_LOCALINSTALL="sudo yum -y --nogpgcheck localinstall" + ;; + *) fatal "Unknown ID in /etc/os-release: $PROVISIONING_OS_ID" ;; + esac + ;; + Darwin) + PROVISIONING_OS=macos + PROVISIONING_OS_ID=macos + CMD_PKG_INSTALL="brew install" + CMD_PKG_LOCALINSTALL="echo 'TODO how to install a package file on macOS'" + ;; + *) + fatal "Unknown system in uname: $uname_s" 42 + ;; + esac + + uname_m="$(uname -m)" + case "$uname_m" in + x86_64|amd64) PROVISIONING_ARCH=amd64 ;; + i[3456]86) PROVISIONING_ARCH=x86 ;; + *) fatal "Unknown architecture in uname: $uname_m" 43 ;; + esac + + CMD_INSTALL="sudo install" +} + +set_common_environment + +set_dry_run () { + if [ x"$PROVISIONING_DRY_RUN" != x ] + then + CMD_PKG_INSTALL="echo DRYRUN: $CMD_PKG_INSTALL" + CMD_PKG_LOCALINSTALL="echo DRYRUN: $CMD_PKG_LOCALINSTALL" + CMD_INSTALL="echo DRYRUN: $CMD_INSTALL" + fi +} + +set_dry_run diff --git a/coin/provisioning/common/unix/telegraf_install.sh b/coin/provisioning/common/unix/telegraf_install.sh new file mode 100755 index 00000000..1412650e --- /dev/null +++ b/coin/provisioning/common/unix/telegraf_install.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +# This script installs telegraf and ioping and our script telegraf-ioping.sh +# to the /usr/bin directory. +# +# The reasons we don't install to /usr/local/bin are: +# 1. On SLES and RHEL, the PATH of sudo (secure_path setting in /etc/sudoers) +# does not include /usr/local/bin. +# 2. On macOS /usr/local/bin does not even exist early in provisioning. + + +######################## BOILERPLATE ########################### +set -e + + +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh + +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh + +is_script_executed telegraf_install.sh \ + || fatal "Script telegraf_install.sh should be executed, not sourced" + +################################################################ + + +[ "$PROVISIONING_OS" = linux ] \ + && ioping_sha256=259abf04bcb84f4126ff97c04b6651e1cf5ea6d8a9ff364c769a26c95b6eeb44 \ + || ioping_sha256=55de6a2f1a5343e0ce8df31d82d47a9e79c7e612edbc6dfb39b5fc6fb358b2e3 +DownloadURL "http://ci-files01-hki.intra.qt.io/input/ioping/ioping.${PROVISIONING_OS}-${PROVISIONING_ARCH}" \ + '' "$ioping_sha256" ioping +$CMD_INSTALL -m 755 ./ioping /usr/bin/ +rm -f ioping + +# 2. Install custom ioping monitoring script +$CMD_INSTALL -m 755 "$PROVISIONING_DIR"/common/shared/telegraf/telegraf-ioping.sh /usr/bin/ + +# 3. Download and install telegraf + +[ x"$PROVISIONING_OS" = xmacos ] && os=darwin || os=linux +[ x"$PROVISIONING_ARCH" = xx86 ] && arch=i386 || arch=amd64 +package_filename=telegraf-1.12.1_${os}_${arch}.tar.gz +package_sha256_list="$PROVISIONING_DIR"/common/shared/telegraf/telegraf_packages.sha256.txt +package_sha256=$(sed -n "s/.*$package_filename *//p" "$package_sha256_list") + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/telegraf/"$package_filename" \ + https://dl.influxdata.com/telegraf/releases/"$package_filename" \ + "$package_sha256" \ + telegraf.tar.gz + +tar -xzf ./telegraf.tar.gz -C /tmp +telegraf_binary=$(find /tmp/telegraf* -name telegraf -type f | grep /bin/ | head -1) +$CMD_INSTALL -m 755 "$telegraf_binary" /usr/bin/ +rm -rf /tmp/telegraf* + +# 4. Edit config file with passwords +"$PROVISIONING_DIR"/common/unix/telegraf_password.sh + +# 5. Start telegraf in background (-b) and with retaining the environment (-E) +# in order to report as hostname = $COIN_UNIQUE_JOB_ID. +/usr/bin/sudo -b -E telegraf --config /etc/telegraf-coin.conf >/dev/null 2>&1 + +echo DONE: "Installed and started telegraf: $package_filename" diff --git a/coin/provisioning/common/unix/telegraf_password.sh b/coin/provisioning/common/unix/telegraf_password.sh new file mode 100755 index 00000000..efbed142 --- /dev/null +++ b/coin/provisioning/common/unix/telegraf_password.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +######################## BOILERPLATE ########################### +set -e + + +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh + +is_script_executed telegraf_password.sh \ + || fatal "Script telegraf_password.sh should be executed, not sourced, to avoid leaking secrets in the logs" +# Avoid leaking secrets in the logs +set +x +################################################################ + + +# Provisioning should run even without the secrets repository +influxdb_password=$(cut -d : -f 2 ~qt/work/influxdb/coin_vms_writer.auth) \ + || influxdb_password=no_password_provided + +rm -f ~qt/work/influxdb/coin_vms_writer.auth +sed "s|COIN_VMS_WRITER_PASS|$influxdb_password|" \ + "$PROVISIONING_DIR"/common/"$PROVISIONING_OS"/telegraf-coin.conf \ + > .telegraf-coin.conf.final +$CMD_INSTALL -m 600 .telegraf-coin.conf.final /etc/telegraf-coin.conf +rm -f .telegraf-coin.conf.final diff --git a/coin/provisioning/common/windows/helpers.ps1 b/coin/provisioning/common/windows/helpers.ps1 index 0f39beed..3ccb42ac 100644 --- a/coin/provisioning/common/windows/helpers.ps1 +++ b/coin/provisioning/common/windows/helpers.ps1 @@ -3,7 +3,7 @@ function Verify-Checksum Param ( [string]$File=$(throw("You must specify a filename to get the checksum of.")), [string]$Expected=$(throw("Checksum required")), - [ValidateSet("sha1","md5")][string]$Algorithm="sha1" + [ValidateSet("sha256","sha1","md5")][string]$Algorithm="sha1" ) Write-Host "Verifying checksum of $File" $fs = new-object System.IO.FileStream $File, "Open" diff --git a/coin/provisioning/common/windows/install_telegraf.ps1 b/coin/provisioning/common/windows/install_telegraf.ps1 new file mode 100644 index 00000000..258c6c59 --- /dev/null +++ b/coin/provisioning/common/windows/install_telegraf.ps1 @@ -0,0 +1,68 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + +. "$PSScriptRoot\..\..\common\windows\helpers.ps1" + + +# Install a specific version of Telegraf +# https://github.com/influxdata/telegraf/releases + +$version = "1.12.1" + +if (Is64BitWinHost) { + $arch = "amd64" + $sha256 = "88d8c8146b561f43726e61688b42d0b5ab88fc22607b94df5a2c8245bbb27e29" +} else { + $arch = "i386" + $sha256 = "d2dea67a28fd7a9e21395d14f0b7ce1b6413e85cc9ffd89cc7ff8f35e8af5837" +} + +$filename = "telegraf-" + $version + "_windows_" + $arch + ".zip" + +$url_cache = "\\ci-files01-hki.intra.qt.io\provisioning\windows\" + $filename +$url_official = "https://dl.influxdata.com/telegraf/releases/" + $filename +$tempfile = "C:\Windows\Temp\" + $filename + +Write-Host "Fetching Telegraf $version..." +Download $url_official $url_cache $tempfile +Verify-Checksum $tempfile $sha256 sha256 + +Write-Host "Installing telegraf.exe under C:\Utils\telegraf" +Extract-7Zip $tempfile C:\Utils "telegraf" +Copy-Item "$PSScriptRoot\..\..\common\windows\telegraf-coin.conf" C:\telegraf-coin.conf + +. "$PSScriptRoot\telegraf_password.ps1" + +Start-Process -FilePath C:\Utils\telegraf\telegraf.exe -ArgumentList "--config C:\telegraf-coin.conf" + +Write-Output "Telegraf = $version" >> ~\versions.txt diff --git a/coin/provisioning/common/windows/telegraf-coin.conf b/coin/provisioning/common/windows/telegraf-coin.conf new file mode 100644 index 00000000..9a4d22e8 --- /dev/null +++ b/coin/provisioning/common/windows/telegraf-coin.conf @@ -0,0 +1,41 @@ +[global_tags] +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 100 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "5s" + precision = "" + logfile = "C:\\telegraf-coin.log" + hostname = "$COIN_UNIQUE_JOB_ID" + omit_hostname = false +[[outputs.influxdb]] + urls= ["https://qtinfluxdb01.intra.qt.io:8086"] + insecure_skip_verify = true + database = "coin_vms" + skip_database_creation = true + username = "coin_vms_writer" + password = "COIN_VMS_WRITER_PASS" +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] +[[inputs.diskio]] +[[inputs.kernel]] +[[inputs.mem]] +#[[inputs.processes]] +[[inputs.swap]] +[[inputs.system]] +#[[inputs.kernel_vmstat]] # linux-only +[[inputs.netstat]] + + +# Monitor Coin process +[[inputs.procstat]] + exe = "coin-" + pid_finder = "native" diff --git a/coin/provisioning/common/windows/telegraf_password.ps1 b/coin/provisioning/common/windows/telegraf_password.ps1 new file mode 100644 index 00000000..11d04dbc --- /dev/null +++ b/coin/provisioning/common/windows/telegraf_password.ps1 @@ -0,0 +1,39 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: http://www.qt.io/licensing/ +## +## This file is part of the provisioning scripts of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:LGPL21$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see http://www.qt.io/terms-conditions. For further +## information use the contact form at http://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 2.1 or version 3 as published by the Free +## Software Foundation and appearing in the file LICENSE.LGPLv21 and +## LICENSE.LGPLv3 included in the packaging of this file. Please review the +## following information to ensure the GNU Lesser General Public License +## requirements will be met: https://www.gnu.org/licenses/lgpl.html and +## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +## +## As a special exception, The Qt Company gives you certain additional +## rights. These rights are described in The Qt Company LGPL Exception +## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +$colon_file = Get-Content "C:\Users\qt\work\influxdb\coin_vms_writer.auth" +$influxdb_password = $colon_file.Substring($colon_file.LastIndexOf(':') + 1) +$telegraf_conf = "C:\telegraf-coin.conf" +(Get-Content $telegraf_conf) | ForEach-Object { $_.Replace("COIN_VMS_WRITER_PASS", $influxdb_password) } | Out-File -Encoding UTF8 $telegraf_conf +Remove-Item "C:\Users\qt\work\influxdb\coin_vms_writer.auth" -- cgit v1.2.3 From 89f6db8c2232b543aaebfe1ba8d9ba8d3a7fc177 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Thu, 12 Sep 2019 01:12:18 +0200 Subject: Adjust pre-existing script to use the new common.sourced.sh Change-Id: Icf7829d3499110a76b90679ba2d067f80d189e69 Reviewed-by: Heikki Halmet --- coin/provisioning/common/macos/increase_limits.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/macos/increase_limits.sh b/coin/provisioning/common/macos/increase_limits.sh index 666d9067..040c55e8 100755 --- a/coin/provisioning/common/macos/increase_limits.sh +++ b/coin/provisioning/common/macos/increase_limits.sh @@ -39,22 +39,20 @@ # and they sometimes create problems to the build process and telegraf. -# Must have this variable set in the environment -[ x"$PROVISION_DIR" = x ] && exit 38 +set -e + +PROVISIONING_DIR="$(dirname "$0")/../../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh echo "Current limits are:" ulimit -a launchctl limit -sudo cp $PROVISION_DIR/common/macos/limit.maxfiles.plist /Library/LaunchDaemons/ -sudo cp $PROVISION_DIR/common/macos/limit.maxproc.plist /Library/LaunchDaemons/ - -sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist -sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist - -sudo chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist -sudo chmod 644 /Library/LaunchDaemons/limit.maxproc.plist +$CMD_INSTALL -m 644 -o root -g wheel \ + $PROVISIONING_DIR/common/macos/limit.maxfiles.plist \ + $PROVISIONING_DIR/common/macos/limit.maxproc.plist \ + /Library/LaunchDaemons/ # Activate the new limits immediately (not for the current session though) sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist -- cgit v1.2.3 From 4e3a0f79fbe8bea7915b267e132ba1e0d4b67733 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Fri, 20 Sep 2019 13:00:12 +0200 Subject: DownloadURL now works with either wget or curl The reason is that Ubuntu does not have curl at the very beginning of provisioning scripts, and also does not have the repositories configured in order to install it. Change-Id: Ia485c1672f894bc800f199b742d238b2945938e5 Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/DownloadURL.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/unix/DownloadURL.sh b/coin/provisioning/common/unix/DownloadURL.sh index da6d772f..0579451f 100755 --- a/coin/provisioning/common/unix/DownloadURL.sh +++ b/coin/provisioning/common/unix/DownloadURL.sh @@ -46,6 +46,15 @@ command -v sha512sum >/dev/null || alias sha512sum='shasum -a 512' ######################################################################## +Download () { + url="$1" + targetFile="$2" + + command -v curl >/dev/null \ + && curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" \ + || wget --tries 5 -O "$targetFile" "$url" +} + VerifyHash () { file=$1 expectedHash=$2 @@ -87,10 +96,10 @@ DownloadURL () { echo "Skipping download, found and validated existing file: $targetFile" else echo "Downloading from primary URL: $url" - if ! curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" + if ! Download "$url" "$targetFile" then echo "FAIL! to download, trying alternative URL: $url2" 1>&2 - if ! curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url2" + if ! Download "$url" "$targetFile" then echo 'FAIL! to download even from alternative url' 1>&2 return 1 -- cgit v1.2.3 From cfef857815392831dc15075c98c53041689a01a4 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 14 Aug 2019 15:52:56 +0200 Subject: Reduce noise in the provision logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce verbosity in order to shorten logs and make them more readable. In two cases we just remove the verbose flag. In one other case we undo inlining of a huge script in the command line. Because we run the provisioning scripts in trace mode, this caused the script to be logged, and it was rather long and meaningless since it comes from an external project. Change-Id: I078ee3b3ab2db9425204375deb5815b6526a4466 Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/linux/qnx_660.sh | 2 +- coin/provisioning/common/macos/homebrew.sh | 11 +++++++++-- coin/provisioning/common/macos/install-commandlinetools.sh | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/qnx_660.sh b/coin/provisioning/common/linux/qnx_660.sh index 2add6901..eb6fd5f0 100755 --- a/coin/provisioning/common/linux/qnx_660.sh +++ b/coin/provisioning/common/linux/qnx_660.sh @@ -49,7 +49,7 @@ echo "$sha1 $targetFile" | sha1sum --check if [ ! -d "$targetFolder" ]; then mkdir -p $targetFolder fi -sudo tar -C $targetFolder -xvzf $targetFile +sudo tar -C $targetFolder -xzf $targetFile sudo chown -R qt:users "$targetFolder"/"$folderName" # Verify that we have last file in tar diff --git a/coin/provisioning/common/macos/homebrew.sh b/coin/provisioning/common/macos/homebrew.sh index f7042de3..715477c0 100755 --- a/coin/provisioning/common/macos/homebrew.sh +++ b/coin/provisioning/common/macos/homebrew.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash -# Will install homebrew package manager for macOS -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" Date: Wed, 2 Oct 2019 19:04:25 +0200 Subject: Fix provisioning script for RHEL 6 It seems RHEL 6.6 does not provide /etc/os-release. Change-Id: I01ba2bb17b89c01f9991944a11eeaa9c2eba6f20 Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/common.sourced.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh index a52880b2..7fb267a6 100644 --- a/coin/provisioning/common/unix/common.sourced.sh +++ b/coin/provisioning/common/unix/common.sourced.sh @@ -74,6 +74,25 @@ is_script_executed common.sourced.sh \ && fatal "Script common.sourced.sh should always be sourced, not executed" +_detect_linux_OS_ID () { + if [ -f /etc/os-release ] + then + . /etc/os-release + PROVISIONING_OS_ID="$ID" + elif [ -f /etc/redhat-release ] + then + case "$(cat /etc/redhat-release)" in + "Red Hat Enterprise Linux"*) + PROVISIONING_OS_ID="rhel" + ;; + "CentOS Linux"*) + PROVISIONING_OS_ID="centos" + ;; + *) fatal "Unknown string in /etc/redhat-release" ;; + esac + fi +} + set_common_environment () { # Unfortunately we can't find the provisioning directory from a sourced # script in a portable way @@ -86,8 +105,7 @@ set_common_environment () { case "$uname_s" in Linux) PROVISIONING_OS=linux - . /etc/os-release - PROVISIONING_OS_ID="$ID" + _detect_linux_OS_ID case "$PROVISIONING_OS_ID" in suse|sles|opensuse*) CMD_PKG_INSTALL="sudo zypper -nq install" -- cgit v1.2.3 From 869a9b3ae7135158247113921c2dfc6cbe7ad6ce Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 2 Oct 2019 19:05:15 +0200 Subject: Fix shell function is_script_executed It used to return error when tested inside a login shell, because $0 is "-bash" and `basename` was trying to interpret -b as an option. Change-Id: I5faa2dc01e50c7d5bcfc0ab1fcfd910b2538afc2 Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/common.sourced.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/unix/common.sourced.sh b/coin/provisioning/common/unix/common.sourced.sh index 7fb267a6..aca5dd04 100644 --- a/coin/provisioning/common/unix/common.sourced.sh +++ b/coin/provisioning/common/unix/common.sourced.sh @@ -65,8 +65,10 @@ fatal () { fi } +# Takes one argument which should be the filename of this script. Returns true +# if the script is being sourced, false if the script is being executed. is_script_executed () { - [ x"$(basename "$0")" = x"$1" ] + [ x"$(basename $(echo "$0" | sed s/^-//))" = x"$1" ] } -- cgit v1.2.3 From dd9a3883aa93277c429b83dcaad20a2a9aeb7fab Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Thu, 3 Oct 2019 12:44:53 +0200 Subject: Ensure provisioning works even without the secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I3869e881341a719da8a351095c3bf314c2fa2f34 Reviewed-by: Mårten Nordheim --- coin/provisioning/common/windows/telegraf_password.ps1 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/telegraf_password.ps1 b/coin/provisioning/common/windows/telegraf_password.ps1 index 11d04dbc..773ec866 100644 --- a/coin/provisioning/common/windows/telegraf_password.ps1 +++ b/coin/provisioning/common/windows/telegraf_password.ps1 @@ -31,9 +31,16 @@ ## ############################################################################# +$auth_file = "C:\Users\qt\work\influxdb\coin_vms_writer.auth" + +# Provisioning should run even without the secrets repository +if (Test-Path $auth_file) { + $auth_content = Get-Content $auth_file + $influxdb_password = $auth_content.Substring($auth_content.LastIndexOf(':') + 1) + Remove-Item $auth_file +} else { + $influxdb_password = "no_password_provided" +} -$colon_file = Get-Content "C:\Users\qt\work\influxdb\coin_vms_writer.auth" -$influxdb_password = $colon_file.Substring($colon_file.LastIndexOf(':') + 1) $telegraf_conf = "C:\telegraf-coin.conf" (Get-Content $telegraf_conf) | ForEach-Object { $_.Replace("COIN_VMS_WRITER_PASS", $influxdb_password) } | Out-File -Encoding UTF8 $telegraf_conf -Remove-Item "C:\Users\qt\work\influxdb\coin_vms_writer.auth" -- cgit v1.2.3 From e4ae752c0d460cb9a8b3166664319f706f1a1b1a Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Wed, 18 Sep 2019 15:09:52 +0300 Subject: Provisioning: Make sure that Configure and Make will pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During powershell process error is not caught and provisioning continues even if the exitcode is something else that '0' Task-number: QTQAINFRA-3214 Change-Id: I91116a78d0c4defe6ba4287913c2394e199be958 Reviewed-by: Tony Sarajärvi Reviewed-by: Mårten Nordheim --- .../common/windows/android-openssl.ps1 | 23 +++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/android-openssl.ps1 b/coin/provisioning/common/windows/android-openssl.ps1 index 981fa51d..c3c3156a 100644 --- a/coin/provisioning/common/windows/android-openssl.ps1 +++ b/coin/provisioning/common/windows/android-openssl.ps1 @@ -1,6 +1,6 @@ ############################################################################ ## -## Copyright (C) 2018 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -65,8 +65,25 @@ Remove-Item -Path $zip Write-Host "Configuring OpenSSL $version for Android..." Push-Location $destination # $ must be escaped in powershell... -Start-Process -NoNewWindow -Wait -ErrorAction Stop -FilePath "$msys_bash" -ArgumentList ("-lc", "`"pushd $openssl_path; ANDROID_NDK_HOME=$ndkPath PATH=${cc_path}:`$PATH CC=clang $openssl_path/Configure shared android-arm`"") -Start-Process -NoNewWindow -Wait -ErrorAction Stop -FilePath "$msys_bash" -ArgumentList ("-lc", "`"pushd $openssl_path; ANDROID_NDK_HOME=$ndkPath PATH=${cc_path}:`$PATH CC=clang make -f $openssl_path/Makefile build_generated`"") + +function CheckExitCode { + + param ( + $p + ) + + if ($p.ExitCode) { + Write-host "Process failed with exit code: $($p.ExitCode)" + exit 1 + } +} + +$configure = Start-Process -NoNewWindow -Wait -PassThru -ErrorAction Stop -FilePath "$msys_bash" -ArgumentList ("-lc", "`"pushd $openssl_path; ANDROID_NDK_HOME=$ndkPath PATH=${cc_path}:`$PATH CC=clang $openssl_path/Configure shared android-arm`"") +CheckExitCode $configure + +$make = Start-Process -NoNewWindow -Wait -PassThru -ErrorAction Stop -FilePath "$msys_bash" -ArgumentList ("-lc", "`"pushd $openssl_path; ANDROID_NDK_HOME=$ndkPath PATH=${cc_path}:`$PATH CC=clang make -f $openssl_path/Makefile build_generated`"") +CheckExitCode $make + Pop-Location Set-EnvironmentVariable "OPENSSL_ANDROID_HOME" "$destination" -- cgit v1.2.3 From d02f0a913e26aa27f3a6b6f275a24b89a6b4388c Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Thu, 3 Oct 2019 10:16:15 +0300 Subject: Provisioning: Hide windows squish installation output from the log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will prevent the output of unwanted information Change-Id: I5f1824a6bbe98c673ef04dc2daa0db0536d27828 Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/windows/squishInstall.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/squishInstall.ps1 b/coin/provisioning/common/windows/squishInstall.ps1 index de04d242..2cc45785 100644 --- a/coin/provisioning/common/windows/squishInstall.ps1 +++ b/coin/provisioning/common/windows/squishInstall.ps1 @@ -1,6 +1,6 @@ ############################################################################# ## -## Copyright (C) 2018 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Contact: http://www.qt.io/licensing/ ## ## This file is part of the provisioning scripts of the Qt Toolkit. @@ -73,7 +73,9 @@ Function DownloadAndInstallSquish { Write-Host "Fetching from URL $squishUrl" Copy-Item "$SquishUrl" "$SquishInstaller" Write-Host "Installing Squish" - Run-Executable "$SquishInstaller" "$SquishParameters" + $stdoutFile = [System.IO.Path]::GetTempFileName() + $stderrFile = [System.IO.Path]::GetTempFileName() + Start-Process -FilePath "$SquishInstaller" -Wait -ArgumentList $SquishParameters -PassThru -RedirectStandardOutput $stdoutFile -RedirectStandardError $stderrFile | Out-Null Remove-Item -Path $SquishInstaller if ("$bit" -eq "win64") { if ($squishPackage.StartsWith("mingw")) { -- cgit v1.2.3 From 3f9673d1090feecd9e849ca2547f3024e323777a Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Fri, 11 Oct 2019 12:21:58 +0300 Subject: Provisioning: Update Cmake to version 3.7.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QtCreator requires Cmake minimal version to be at least 3.7.0 Task-number: QTQAINFRA-3272 Change-Id: Id7f2d38ad0581c1aead4b5bac860ac193488ae1b Reviewed-by: Tony Sarajärvi Reviewed-by: Nikolai Kosjar --- coin/provisioning/common/linux/cmake_linux.sh | 10 +++++----- coin/provisioning/common/windows/cmake.ps1 | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/cmake_linux.sh b/coin/provisioning/common/linux/cmake_linux.sh index 2b8d71d0..a3b2a9cc 100755 --- a/coin/provisioning/common/linux/cmake_linux.sh +++ b/coin/provisioning/common/linux/cmake_linux.sh @@ -33,7 +33,7 @@ ## ############################################################################# -# This script installs CMake 3.6.2 +# This script installs CMake 3.7.2 # CMake is needed for autotests that verify that Qt can be built with CMake @@ -42,10 +42,10 @@ source "${BASH_SOURCE%/*}/../unix/InstallFromCompressedFileFromURL.sh" # shellcheck source=../unix/SetEnvVar.sh source "${BASH_SOURCE%/*}/../unix/SetEnvVar.sh" -version="3.6.2" -PrimaryUrl="http://ci-files01-hki.intra.qt.io/input/cmake/cmake-3.6.2-Linux-x86_64.tar.gz" -AltUrl="https://cmake.org/files/v3.6/cmake-3.6.2-Linux-x86_64.tar.gz" -SHA1="dd9d8d57b66109d4bac6eef9209beb94608a185c" +version="3.7.2" +PrimaryUrl="http://ci-files01-hki.intra.qt.io/input/cmake/cmake-3.7.2-Linux-x86_64.tar.gz" +AltUrl="https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.tar.gz" +SHA1="915bc981aab354821fb9fd28374a720fdb3aa180" targetFolder="/opt/cmake-$version" appPrefix="cmake-$version-Linux-x86_64" diff --git a/coin/provisioning/common/windows/cmake.ps1 b/coin/provisioning/common/windows/cmake.ps1 index ec84d706..9d3d6aef 100644 --- a/coin/provisioning/common/windows/cmake.ps1 +++ b/coin/provisioning/common/windows/cmake.ps1 @@ -33,8 +33,8 @@ . "$PSScriptRoot\helpers.ps1" -$majorminorversion = "3.6" -$version = "3.6.2" +$majorminorversion = "3.7" +$version = "3.7.2" $zip = Get-DownloadLocation ("cmake-" + $version + "-win32-x86.zip") $officialurl = "https://cmake.org/files/v" + $majorminorversion + "/cmake-" + $version + "-win32-x86.zip" @@ -44,7 +44,7 @@ Write-Host "Removing old cmake" Remove-Item "C:\CMake" -Force -Recurse -ErrorAction SilentlyContinue Download $officialurl $cachedurl $zip -Verify-Checksum $zip "541F6E7EFD228E46770B8631FFE57097576E4D4E" +Verify-Checksum $zip "c80c17e858ecfebfaf16fe8af18b174d2600c4e6" Extract-7Zip $zip C: $defaultinstallfolder = "C:\cmake-" + $version + "-win32-x86" -- cgit v1.2.3 From 518e114dc71fb58d175a15afb282ac7f6dff6adc Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Fri, 11 Oct 2019 11:30:38 +0200 Subject: Fix for DownloadURL to try alternative URL Change-Id: I7e9a77815ef8f4c258dda44bda6fdd4eb3736ef4 Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/DownloadURL.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/unix/DownloadURL.sh b/coin/provisioning/common/unix/DownloadURL.sh index 0579451f..ac1fd971 100755 --- a/coin/provisioning/common/unix/DownloadURL.sh +++ b/coin/provisioning/common/unix/DownloadURL.sh @@ -99,9 +99,9 @@ DownloadURL () { if ! Download "$url" "$targetFile" then echo "FAIL! to download, trying alternative URL: $url2" 1>&2 - if ! Download "$url" "$targetFile" + if ! Download "$url2" "$targetFile" then - echo 'FAIL! to download even from alternative url' 1>&2 + echo 'FAIL! to download even from alternative URL' 1>&2 return 1 fi fi -- cgit v1.2.3 From 60befd750829fe5a61b07256f0dc667ea1bd0f2f Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Mon, 30 Sep 2019 10:18:38 +0300 Subject: Provisioning: Disable automatic cleanup from windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task-number: QTQAINFRA-3238 Change-Id: Ie64086b650016adb7129bd4c4592ca987cc1453b Reviewed-by: Daniel Smith Reviewed-by: Dimitrios Apostolou Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/windows/disable-clean-manager.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 coin/provisioning/common/windows/disable-clean-manager.ps1 (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/disable-clean-manager.ps1 b/coin/provisioning/common/windows/disable-clean-manager.ps1 new file mode 100644 index 00000000..6d54e137 --- /dev/null +++ b/coin/provisioning/common/windows/disable-clean-manager.ps1 @@ -0,0 +1,8 @@ +# This script will disable automatic disk cleanup + +. "$PSScriptRoot\helpers.ps1" + +Run-Executable "reg.exe" "ADD `"HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\StorageSense\Parameters\StoragePolicy`" /V 04 /T REG_dWORD /D 0 /F" + +# Maintenance task used by the system to launch a silent auto disk cleanup when running low on free disk space. +DisableSchedulerTask "DiskCleanup\SilentCleanup" -- cgit v1.2.3 From 17550319e50ecff255e6a8ec31919ed90b6a1880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tony=20Saraj=C3=A4rvi?= Date: Tue, 22 Oct 2019 11:34:26 +0000 Subject: Provision gnuwin32's path By provisioning the path to gnuwin32, we can remove the hard coded paths from Coin. Task-number: QTBUG-73422 Change-Id: Ice96fcd992e22ed39b66496403b9eaa99a6865a0 Reviewed-by: Toni Saario Reviewed-by: Heikki Halmet --- coin/provisioning/common/windows/install-gnuwin32.ps1 | 1 + 1 file changed, 1 insertion(+) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/windows/install-gnuwin32.ps1 b/coin/provisioning/common/windows/install-gnuwin32.ps1 index 99862b3c..6acede79 100644 --- a/coin/provisioning/common/windows/install-gnuwin32.ps1 +++ b/coin/provisioning/common/windows/install-gnuwin32.ps1 @@ -44,3 +44,4 @@ Verify-Checksum "$temp\$zipPackage" "d7a34a385ccde2374b8a2ca3369e5b8a1452c5a5" Extract-7Zip "$temp\$zipPackage" C:\Utils Write-Output "$prog qt5 commit sha = 98c4f1bbebfb3cc6d8e031d36fd1da3c19e634fb" >> ~\versions.txt +Prepend-Path "C:\Utils\gnuwin32\bin" -- cgit v1.2.3 From eb0ef419ec91ed580b13d54bf480087b7d33c2e1 Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Fri, 25 Oct 2019 15:20:53 +0300 Subject: Add allowUntrusted parameter for command line tools and refresh Xcode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apple has signed their packages with certificate which has been expired 24.11.2019. Packages are unusable without allowUntrusted parameter Also Xcode package for macOS10.13 needs to be refetched from Apple's download to local cache Task-number: QTQAINFRA-3317 Change-Id: Iebbe81ee0c360e5f0b49d422b38fe1a148424891 Reviewed-by: Tony Sarajärvi --- coin/provisioning/common/macos/install-commandlinetools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/macos/install-commandlinetools.sh b/coin/provisioning/common/macos/install-commandlinetools.sh index b9f81534..cd7838b6 100755 --- a/coin/provisioning/common/macos/install-commandlinetools.sh +++ b/coin/provisioning/common/macos/install-commandlinetools.sh @@ -50,7 +50,7 @@ function InstallCommandLineTools { hdiutil attach "/tmp/$packageName" cd "/Volumes/Command Line Developer Tools" echo "Installing" - sudo installer -pkg ./*.pkg -target / + sudo installer -pkg ./*.pkg -target / -allowUntrusted cd / # Let's fait for 5 second before unmounting. Sometimes resource is busy and cant be unmounted sleep 3 -- cgit v1.2.3 From 2eab420e807d5fa4e39d4437879b9a35a2cdc88e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tony=20Saraj=C3=A4rvi?= Date: Thu, 31 Oct 2019 13:32:28 +0000 Subject: Remove more of update notifiers Change-Id: I507f584ef82b5f41f6b1cbc22fdd4ff6eff5e911 Reviewed-by: Heikki Halmet --- coin/provisioning/common/linux/remove-update_notifier.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/remove-update_notifier.sh b/coin/provisioning/common/linux/remove-update_notifier.sh index 152d7fa8..4ee32d61 100755 --- a/coin/provisioning/common/linux/remove-update_notifier.sh +++ b/coin/provisioning/common/linux/remove-update_notifier.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -sudo apt -q -y remove update-notifier update-manager +sudo apt -q -y remove update-notifier update-manager python3-distupgrade python3-update-manager ubuntu-release-upgrader-core update-manager-core -- cgit v1.2.3 From 52fe759ebacd4d58945f0dd01b3aecf321e20375 Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Wed, 18 Sep 2019 13:43:22 +0300 Subject: Provisioning: Upgrade OpenSSL version to 1.1.1d MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plus removing unused file install_openssl_111a.sh Task-number: QTQAINFRA-3224 Change-Id: If5f9ee12e118e0329eb27f8d3ca9829011874382 Reviewed-by: Tony Sarajärvi --- .../common/linux/openssl_for_android_linux.sh | 4 +- .../common/macos/install_openssl_111a.sh | 78 ---------------------- coin/provisioning/common/unix/install-openssl.sh | 4 +- .../common/windows/android-openssl.ps1 | 4 +- coin/provisioning/common/windows/openssl.ps1 | 6 +- 5 files changed, 9 insertions(+), 87 deletions(-) delete mode 100755 coin/provisioning/common/macos/install_openssl_111a.sh (limited to 'coin/provisioning/common') diff --git a/coin/provisioning/common/linux/openssl_for_android_linux.sh b/coin/provisioning/common/linux/openssl_for_android_linux.sh index d833975f..4860c01f 100755 --- a/coin/provisioning/common/linux/openssl_for_android_linux.sh +++ b/coin/provisioning/common/linux/openssl_for_android_linux.sh @@ -53,11 +53,11 @@ else rm -rf "$exports_file" fi -version="1.1.1b" +version="1.1.1d" officialUrl="https://www.openssl.org/source/openssl-$version.tar.gz" cachedUrl="http://ci-files01-hki.intra.qt.io/input/openssl/openssl-$version.tar.gz" targetFile="/tmp/openssl-$version.tar.gz" -sha="e9710abf5e95c48ebf47991b10cbb48c09dae102" +sha="056057782325134b76d1931c48f2c7e6595d7ef4" opensslHome="${HOME}/openssl/android/openssl-${version}" DownloadURL "$cachedUrl" "$officialUrl" "$sha" "$targetFile" mkdir -p "${HOME}/openssl/android/" diff --git a/coin/provisioning/common/macos/install_openssl_111a.sh b/coin/provisioning/common/macos/install_openssl_111a.sh deleted file mode 100755 index cd6cd9e8..00000000 --- a/coin/provisioning/common/macos/install_openssl_111a.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: http://www.qt.io/licensing/ -## -## This file is part of the provisioning scripts of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:LGPL21$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see http://www.qt.io/terms-conditions. For further -## information use the contact form at http://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 2.1 or version 3 as published by the Free -## Software Foundation and appearing in the file LICENSE.LGPLv21 and -## LICENSE.LGPLv3 included in the packaging of this file. Please review the -## following information to ensure the GNU Lesser General Public License -## requirements will be met: https://www.gnu.org/licenses/lgpl.html and -## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -## -## As a special exception, The Qt Company gives you certain additional -## rights. These rights are described in The Qt Company LGPL Exception -## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -## -## $QT_END_LICENSE$ -## -############################################################################# - -# This script install OpenSSL - -set -ex - -# shellcheck source=../unix/InstallFromCompressedFileFromURL.sh -source "${BASH_SOURCE%/*}/../unix/InstallFromCompressedFileFromURL.sh" -# shellcheck source=../unix/SetEnvVar.sh -source "${BASH_SOURCE%/*}/../unix/SetEnvVar.sh" - -opensslVersion="1.1.1a" -opensslFile="openssl-$opensslVersion.tar.gz" -opensslDlUrl="http://ci-files01-hki.intra.qt.io/input/openssl/$opensslFile" -opensslAltDlUrl="https://www.openssl.org/source/$opensslFile" -opensslSha1="8fae27b4f34445a5500c9dc50ae66b4d6472ce29" - -# Below target location has been hard coded into Coin. -# QTQAINFRA-1195 -openssl_install_dir=/usr/local/openssl-$opensslVersion -opensslTargetLocation="/usr/local/opt/openssl" - -InstallFromCompressedFileFromURL "$opensslDlUrl" "$opensslAltDlUrl" "$opensslSha1" "/tmp/openssl-$opensslVersion" "openssl-$opensslVersion" -cd "/tmp/openssl-$opensslVersion" -sudo ./Configure --prefix=$openssl_install_dir shared no-ssl3-method enable-ec_nistp_64_gcc_128 darwin64-x86_64-cc "-Wa,--noexecstack" - -sudo make install_sw install_ssldirs - -path=$(echo "$opensslTargetLocation" | sed -E 's/(.*)\/.*$/\1/') -sudo mkdir -p "$path" -sudo ln -s $openssl_install_dir $opensslTargetLocation - -SetEnvVar "PATH" "\"$opensslTargetLocation/bin:\$PATH\"" -SetEnvVar "MANPATH" "\"$opensslTargetLocation/share/man:\$MANPATH\"" - -SetEnvVar "OPENSSL_DIR" "\"$openssl_install_dir\"" -SetEnvVar "OPENSSL_INCLUDE" "\"$openssl_install_dir/include\"" -SetEnvVar "OPENSSL_LIB" "\"$openssl_install_dir/lib\"" - -security find-certificate -a -p /Library/Keychains/System.keychain | sudo tee -a $opensslTargetLocation/ssl/cert.pem > /dev/null -security find-certificate -a -p /System/Library/Keychains/SystemRootCertificates.keychain | sudo tee -a $opensslTargetLocation/ssl/cert.pem > /dev/null - -sudo rm -rf /tmp/openssl-$opensslVersion - -echo "OpenSSL = $opensslVersion" >> ~/versions.txt diff --git a/coin/provisioning/common/unix/install-openssl.sh b/coin/provisioning/common/unix/install-openssl.sh index 01a20935..08a3390b 100755 --- a/coin/provisioning/common/unix/install-openssl.sh +++ b/coin/provisioning/common/unix/install-openssl.sh @@ -42,11 +42,11 @@ source "${BASH_SOURCE%/*}/../unix/DownloadURL.sh" # shellcheck source=../unix/SetEnvVar.sh source "${BASH_SOURCE%/*}/../unix/SetEnvVar.sh" -version="1.1.1b" +version="1.1.1d" officialUrl="https://www.openssl.org/source/openssl-$version.tar.gz" cachedUrl="http://ci-files01-hki.intra.qt.io/input/openssl/openssl-$version.tar.gz" targetFile="/tmp/openssl-$version.tar.gz" -sha="e9710abf5e95c48ebf47991b10cbb48c09dae102" +sha="056057782325134b76d1931c48f2c7e6595d7ef4" opensslHome="${HOME}/openssl-${version}" opensslSource="${opensslHome}-src" DownloadURL "$cachedUrl" "$officialUrl" "$sha" "$targetFile" diff --git a/coin/provisioning/common/windows/android-openssl.ps1 b/coin/provisioning/common/windows/android-openssl.ps1 index c3c3156a..e4306584 100644 --- a/coin/provisioning/common/windows/android-openssl.ps1 +++ b/coin/provisioning/common/windows/android-openssl.ps1 @@ -45,9 +45,9 @@ if (Is64BitWinHost) { # Msys need to be installed to target machine # More info and building instructions can be found from http://doc.qt.io/qt-5/opensslsupport.html -$version = "1.1.1b" +$version = "1.1.1d" $zip = Get-DownloadLocation ("openssl-$version.tar.gz") -$sha1 = "e9710abf5e95c48ebf47991b10cbb48c09dae102" +$sha1 = "056057782325134b76d1931c48f2c7e6595d7ef4" $destination = "C:\Utils\openssl-android-master" # msys unix style paths diff --git a/coin/provisioning/common/windows/openssl.ps1 b/coin/provisioning/common/windows/openssl.ps1 index 2f21db37..891f5db7 100644 --- a/coin/provisioning/common/windows/openssl.ps1 +++ b/coin/provisioning/common/windows/openssl.ps1 @@ -36,7 +36,7 @@ # This script installs OpenSSL $version. # Both x86 and x64 versions needed when x86 integrations are done on x64 machine -$version = "1_1_1b" +$version = "1_1_1d" $packagex64 = "C:\Windows\Temp\Win64OpenSSL-$version.exe" $packagex86 = "C:\Windows\Temp\Win32OpenSSL-$version.exe" @@ -47,7 +47,7 @@ if (Is64BitWinHost) { $installFolder = "C:\openssl" $externalUrl = "https://slproweb.com/download/Win64OpenSSL-$version.exe" $internalUrl = "\\ci-files01-hki.intra.qt.io\provisioning\openssl\Win64OpenSSL-$version.exe" - $sha1 = "5b50819dd84aa9219e0cad9cdddf78285bdd1bbb" + $sha1 = "df9c40e9e82edf693c15302cbc8e8a057857a13c" Write-Host "Fetching from URL ..." Download $externalUrl $internalUrl $packagex64 @@ -74,7 +74,7 @@ if (Is64BitWinHost) { $externalUrl = "https://slproweb.com/download/Win32OpenSSL-$version.exe" $internalUrl = "\\ci-files01-hki.intra.qt.io\provisioning\openssl\Win32OpenSSL-$version.exe" -$sha1 = "1ae5ad4fe5dae01dd056274979cce26945c9e86c" +$sha1 = "5c49ccd3a5d17f63d25fb819963978f50c7966d0" Write-Host "Fetching from URL ..." Download $externalUrl $internalUrl $packagex86 -- cgit v1.2.3