From f9f9e0944d0e0e7d04dd3b08479f867ab7fd638c Mon Sep 17 00:00:00 2001 From: Dominik Holland Date: Mon, 16 Jul 2018 15:13:05 +0200 Subject: Download and cache needed python wheels during provisioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of downloading the packages during the compile/configure step setup a wheel cache folder during the provisioning phase Task-number: AUTOSUITE-195 Change-Id: I465f1be7cdd351e7680dcd8ae22d5e97ddf8cb2d Reviewed-by: Jędrzej Nowacki --- coin/provisioning/common/macos/python3.sh | 6 +++++- coin/provisioning/common/shared/requirements.txt | 19 +++++++++++++++++++ coin/provisioning/common/windows/python3.ps1 | 7 ++++++- .../qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh | 8 +++++++- .../qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh | 7 +++++++ .../qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh | 8 +++++++- 6 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 coin/provisioning/common/shared/requirements.txt diff --git a/coin/provisioning/common/macos/python3.sh b/coin/provisioning/common/macos/python3.sh index 545a6d02..35bccc93 100755 --- a/coin/provisioning/common/macos/python3.sh +++ b/coin/provisioning/common/macos/python3.sh @@ -52,10 +52,14 @@ InstallPKGFromURL "$PrimaryUrl" "$AltUrl" "$SHA1" "$DestDir" InstallPip python3.6 -/Library/Frameworks/Python.framework/Versions/3.6/bin/pip3 install virtualenv +/Library/Frameworks/Python.framework/Versions/3.6/bin/pip3 install virtualenv wheel SetEnvVar "PYTHON3_PATH" "/Library/Frameworks/Python.framework/Versions/3.6/bin" SetEnvVar "PIP3_PATH" "/Library/Frameworks/Python.framework/Versions/3.6/bin" +# Install all needed packages in a special wheel cache directory +/Library/Frameworks/Python.framework/Versions/3.6/bin/pip3 wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../shared/requirements.txt +SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels" + echo "python3 = 3.6.1" >> ~/versions.txt diff --git a/coin/provisioning/common/shared/requirements.txt b/coin/provisioning/common/shared/requirements.txt new file mode 100644 index 00000000..12380d85 --- /dev/null +++ b/coin/provisioning/common/shared/requirements.txt @@ -0,0 +1,19 @@ +# This file holds all python3 packages need for a certain qt module +# When adding new packages here, first add a comment with the name of the module +# Then all the packages. Duplicates are wanted as every module should install +# all packages it needs and shouldn't depend on a package from another module +# +# qtivi +antlr4-python3-runtime==4.7.1 +argh==0.26.2 +click==6.7 +coloredlogs==10.0 +humanfriendly==4.15.1 +Jinja2==2.10 +MarkupSafe==1.0 +path.py==11.0.1 +pathtools==0.1.2 +PyYAML==3.13 +six==1.11.0 +typing==3.6.4 +watchdog==0.8.3 diff --git a/coin/provisioning/common/windows/python3.ps1 b/coin/provisioning/common/windows/python3.ps1 index 473fe65c..e4ab667f 100644 --- a/coin/provisioning/common/windows/python3.ps1 +++ b/coin/provisioning/common/windows/python3.ps1 @@ -79,7 +79,12 @@ if (IsProxyEnabled) { Write-Host "Using proxy ($proxy) with pip" $pip_args = "--proxy=$proxy" } -Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv" +Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv wheel" + +# Install all needed packages in a special wheel cache directory +$python3_wheel_dir="$install_path\python3-wheels" +Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args wheel --wheel-dir $python3_wheel_dir -r $PSScriptRoot\..\shared\requirements.txt" +Set-EnvironmentVariable "PYTHON3_WHEEL_CACHE" "$python3_wheel_dir" Write-Output "Python3-$archVer = $version" >> ~/versions.txt diff --git a/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh b/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh index a0fa62a9..6527625e 100755 --- a/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh +++ b/coin/provisioning/qtci-linux-RHEL-7.4-x86_64/04-install-packages.sh @@ -103,4 +103,10 @@ sudo ln -s /opt/rh/rh-python36/root/usr/bin/pip3 /usr/local/bin/pip3 # We shouldn't use yum to install virtualenv. The one found from package repo is not # working, but we can use installed pip sudo pip install --upgrade pip -sudo pip install virtualenv +sudo pip install virtualenv wheel + +# Install all needed packages in a special wheel cache directory +pip wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt + +source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh" +SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels" diff --git a/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh b/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh index e78cba17..aff325bb 100755 --- a/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh +++ b/coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh @@ -110,6 +110,7 @@ installPackages+=(python-pip) installPackages+=(python3-dev) installPackages+=(python3-pip) installPackages+=(python3-virtualenv) +installPackages+=(python3-wheel) # Needed to be able to build Yocto installPackages+=(chrpath) installPackages+=(gawk) @@ -162,3 +163,9 @@ echo "Running update for apt" sudo apt-get update echo "Installing packages" sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install "${installPackages[@]}" + +# Install all needed packages in a special wheel cache directory +pip wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt + +source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh" +SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels" diff --git a/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh index e7cdd4a7..a5494041 100755 --- a/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh +++ b/coin/provisioning/qtci-linux-openSUSE-42.3-x86_64/08-pythondev.sh @@ -9,4 +9,10 @@ sudo pkcon -y refresh sudo pkcon -y install python-devel python-virtualenv # install python3 -sudo pkcon -y install libpython3_4m1_0 python3-base python3 python3-pip python3-devel python3-virtualenv +sudo pkcon -y install libpython3_4m1_0 python3-base python3 python3-pip python3-devel python3-virtualenv python3-wheel + +# Install all needed packages in a special wheel cache directory +pip wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt + +source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh" +SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels" -- cgit v1.2.3