Download and cache needed python wheels during provisioning

Instead of downloading the packages during the compile/configure
step setup a wheel cache folder during the provisioning phase

Task-number: AUTOSUITE-195
Change-Id: I465f1be7cdd351e7680dcd8ae22d5e97ddf8cb2d
Reviewed-by: Jędrzej Nowacki <jedrzej.nowacki@qt.io>

6 files changed, 51 insertions, 4 deletions

diff --git a/coin/provisioning/common/macos/python3.sh b/coin/provisioning/common/macos/python3.sh
index 545a6d02..35bccc93 100755
--- a/coin/provisioning/common/macos/python3.sh
+++ b/coin/provisioning/common/macos/python3.sh
@@ -52,10 +52,14 @@ InstallPKGFromURL "$PrimaryUrl" "$AltUrl" "$SHA1" "$DestDir"
 
 InstallPip python3.6
 
-/Library/Frameworks/Python.framework/Versions/3.6/bin/pip3 install virtualenv
+/Library/Frameworks/Python.framework/Versions/3.6/bin/pip3 install virtualenv wheel
 
 SetEnvVar "PYTHON3_PATH" "/Library/Frameworks/Python.framework/Versions/3.6/bin"
 SetEnvVar "PIP3_PATH" "/Library/Frameworks/Python.framework/Versions/3.6/bin"
 
+# Install all needed packages in a special wheel cache directory
+/Library/Frameworks/Python.framework/Versions/3.6/bin/pip3 wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../shared/requirements.txt
+SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
+
 echo "python3 = 3.6.1" >> ~/versions.txt
 
diff --git a/coin/provisioning/common/shared/requirements.txt b/coin/provisioning/common/shared/requirements.txt
new file mode 100644
index 00000000..12380d85
--- /dev/null
+++ b/coin/provisioning/common/shared/requirements.txt
@@ -0,0 +1,19 @@
+# This file holds all python3 packages need for a certain qt module
+# When adding new packages here, first add a comment with the name of the module
+# Then all the packages. Duplicates are wanted as every module should install
+# all packages it needs and shouldn't depend on a package from another module
+#
+# qtivi
+antlr4-python3-runtime==4.7.1
+argh==0.26.2
+click==6.7
+coloredlogs==10.0
+humanfriendly==4.15.1
+Jinja2==2.10
+MarkupSafe==1.0
+path.py==11.0.1
+pathtools==0.1.2
+PyYAML==3.13
+six==1.11.0
+typing==3.6.4
+watchdog==0.8.3
diff --git a/coin/provisioning/common/windows/python3.ps1 b/coin/provisioning/common/windows/python3.ps1
index 473fe65c..e4ab667f 100644
--- a/coin/provisioning/common/windows/python3.ps1
+++ b/coin/provisioning/common/windows/python3.ps1
@@ -79,7 +79,12 @@ if (IsProxyEnabled) {
     Write-Host "Using proxy ($proxy) with pip"
     $pip_args = "--proxy=$proxy"
 }
-Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv"
+Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv wheel"
+
+# Install all needed packages in a special wheel cache directory
+$python3_wheel_dir="$install_path\python3-wheels"
+Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args wheel --wheel-dir $python3_wheel_dir -r $PSScriptRoot\..\shared\requirements.txt"
+Set-EnvironmentVariable "PYTHON3_WHEEL_CACHE" "$python3_wheel_dir"
 
 Write-Output "Python3-$archVer = $version" >> ~/versions.txt
 
diff --git a/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh b/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh
index a0fa62a9..6527625e 100755
--- a/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh
+++ b/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh
@@ -103,4 +103,10 @@ sudo ln -s /opt/rh/rh-python36/root/usr/bin/pip3 /usr/local/bin/pip3
 # We shouldn't use yum to install virtualenv. The one found from package repo is not
 # working, but we can use installed pip
 sudo pip install --upgrade pip
-sudo pip install virtualenv
+sudo pip install virtualenv wheel
+
+# Install all needed packages in a special wheel cache directory
+pip wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt
+
+source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
+SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
diff --git a/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh b/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh
index e78cba17..aff325bb 100755
--- a/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh
+++ b/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh
@@ -110,6 +110,7 @@ installPackages+=(python-pip)
 installPackages+=(python3-dev)
 installPackages+=(python3-pip)
 installPackages+=(python3-virtualenv)
+installPackages+=(python3-wheel)
 # Needed to be able to build Yocto
 installPackages+=(chrpath)
 installPackages+=(gawk)
@@ -162,3 +163,9 @@ echo "Running update for apt"
 sudo apt-get update
 echo "Installing packages"
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install "${installPackages[@]}"
+
+# Install all needed packages in a special wheel cache directory
+pip wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt
+
+source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
+SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
diff --git a/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh
index e7cdd4a7..a5494041 100755
--- a/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh
+++ b/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh
@@ -9,4 +9,10 @@ sudo pkcon -y refresh
 sudo pkcon -y install python-devel python-virtualenv
 
 # install python3
-sudo pkcon -y install libpython3_4m1_0 python3-base python3 python3-pip python3-devel python3-virtualenv
+sudo pkcon -y install libpython3_4m1_0 python3-base python3 python3-pip python3-devel python3-virtualenv python3-wheel
+
+# Install all needed packages in a special wheel cache directory
+pip wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt
+
+source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
+SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"