From 8d04f876e73ae16a6381ba45bbe51377ad23a60f Mon Sep 17 00:00:00 2001 From: Oliver Wolff Date: Mon, 19 Mar 2018 13:04:11 +0100 Subject: windows provisioning: Do not use Windows builtin way of extracting zips The builtin way of Windows for handling zip files is much slower than using 7zip. A small benchmark (not representative, just extracting the android ndk) showed 11 minutes using Extract-Zip vs 2,5 minutes using Extract-7Zip on my machine. Change-Id: I5198aabb5b75860ead687581fc5e368eb75e48ae Reviewed-by: Simon Hausmann --- .../common/windows/3dstudio_3rdparty.ps1 | 2 +- coin/provisioning/common/windows/android.ps1 | 2 +- coin/provisioning/common/windows/cmake.ps1 | 2 +- coin/provisioning/common/windows/helpers.ps1 | 48 +++------------------- coin/provisioning/common/windows/jom.ps1 | 2 +- coin/provisioning/common/windows/mqtt_broker.ps1 | 2 +- coin/provisioning/common/windows/mysql.ps1 | 24 +++++------ coin/provisioning/common/windows/ninja.ps1 | 2 +- coin/provisioning/common/windows/postgresql.ps1 | 18 ++++---- coin/provisioning/common/windows/winrtrunner.ps1 | 2 +- .../qtci-windows-10-x86_64/90-signing-tools.ps1 | 2 +- 11 files changed, 33 insertions(+), 73 deletions(-) (limited to 'coin') diff --git a/coin/provisioning/common/windows/3dstudio_3rdparty.ps1 b/coin/provisioning/common/windows/3dstudio_3rdparty.ps1 index 924d8606..ed408b6c 100644 --- a/coin/provisioning/common/windows/3dstudio_3rdparty.ps1 +++ b/coin/provisioning/common/windows/3dstudio_3rdparty.ps1 @@ -5,7 +5,7 @@ $url = "http://ci-files01-hki.intra.qt.io/input/3rdparty/Qt3DStudio-3rdparty-win Download $url $url $zip Verify-Checksum $zip "08D740D2EFB4CBCDE7D012908B89AA48DE5CD4E1" -Extract-Zip $zip C:\Utils\Qt3DStudio3rdparty +Extract-7Zip $zip C:\Utils\Qt3DStudio3rdparty Remove-Item -Path $zip Set-EnvironmentVariable "QT3DSTUDIO_3RDPARTY_DIR" "C:/Utils/Qt3DStudio3rdparty" diff --git a/coin/provisioning/common/windows/android.ps1 b/coin/provisioning/common/windows/android.ps1 index 8fa19e04..9a09e919 100644 --- a/coin/provisioning/common/windows/android.ps1 +++ b/coin/provisioning/common/windows/android.ps1 @@ -66,7 +66,7 @@ function Install($1, $2, $3, $4) { Download $offcialUrl $cacheUrl $zip Verify-Checksum $zip "$checksum" - Extract-Zip $zip C:\Utils + Extract-7Zip $zip C:\Utils } function SdkUpdate ($1, $2) { diff --git a/coin/provisioning/common/windows/cmake.ps1 b/coin/provisioning/common/windows/cmake.ps1 index c0ead98e..ace68835 100644 --- a/coin/provisioning/common/windows/cmake.ps1 +++ b/coin/provisioning/common/windows/cmake.ps1 @@ -46,7 +46,7 @@ Remove-Item "C:\CMake" -Force -Recurse -ErrorAction SilentlyContinue Download $officialurl $cachedurl $zip Verify-Checksum $zip "541F6E7EFD228E46770B8631FFE57097576E4D4E" -Extract-Zip $zip C: +Extract-7Zip $zip C: $defaultinstallfolder = "C:\cmake-" + $version + "-win32-x86" Rename-Item $defaultinstallfolder C:\CMake diff --git a/coin/provisioning/common/windows/helpers.ps1 b/coin/provisioning/common/windows/helpers.ps1 index 45779c04..a2535b25 100644 --- a/coin/provisioning/common/windows/helpers.ps1 +++ b/coin/provisioning/common/windows/helpers.ps1 @@ -38,7 +38,8 @@ function Extract-7Zip { Param ( [string]$Source, - [string]$Destination + [string]$Destination, + [string]$Filter ) Write-Host "Extracting '$Source' to '$Destination'..." @@ -54,47 +55,10 @@ function Extract-7Zip $zipExe = "7z.exe" } - Run-Executable "$zipExe" "x -y `"-o$Destination`" `"$Source`"" -} - -function Extract-Zip -{ - Param ( - [string]$Source, - [string]$Destination - ) - Write-Host "Extracting '$Source' to '$Destination'..." - - New-Item -ItemType Directory -Force -Path $Destination - $shell = new-object -com shell.application - $zipfile = $shell.Namespace($Source) - $destinationFolder = $shell.Namespace($Destination) - $destinationFolder.CopyHere($zipfile.Items(), 16) -} - -function Extract-Dev-Folders-From-Zip -{ - Param ( - [string]$package, - [string]$zipDir, - [string]$installPath - ) - - $shell = new-object -com shell.application - - Write-Host "Extracting contents of $package" - foreach ($subDir in "lib", "include", "bin", "share") { - $zip = $shell.Namespace($package + "\" + $zipDir + "\" + $subDir) - if ($zip) { - Write-Host "Extracting $subDir from zip archive" - } else { - Write-Host "$subDir is missing from zip archive - skipping" - continue - } - $destDir = $installPath + "\" + $subdir - New-Item $destDir -type directory - $destinationFolder = $shell.Namespace($destDir) - $destinationFolder.CopyHere($zip.Items(), 16) + if ([string]::IsNullOrEmpty($Filter)) { + Run-Executable "$zipExe" "x -y `"-o$Destination`" `"$Source`"" + } else { + Run-Executable "$zipExe" "x -y -aoa `"-o$Destination`" `"$Source`" $Filter" } } diff --git a/coin/provisioning/common/windows/jom.ps1 b/coin/provisioning/common/windows/jom.ps1 index 2344eec7..68db6b17 100644 --- a/coin/provisioning/common/windows/jom.ps1 +++ b/coin/provisioning/common/windows/jom.ps1 @@ -4,7 +4,7 @@ $zip = "c:\users\qt\downloads\jom_1_1_2.zip" Invoke-WebRequest -UseBasicParsing http://download.qt.io/official_releases/jom/jom_1_1_2.zip -OutFile $zip Verify-Checksum $zip "80EE5678E714DE99DDAF5F7593AB04DB1C7928E4" -Extract-Zip $zip C:\Utils\Jom +Extract-7Zip $zip C:\Utils\Jom Set-EnvironmentVariable "CI_JOM_PATH" "C:\Utils\Jom" diff --git a/coin/provisioning/common/windows/mqtt_broker.ps1 b/coin/provisioning/common/windows/mqtt_broker.ps1 index b88f568e..205d9c95 100644 --- a/coin/provisioning/common/windows/mqtt_broker.ps1 +++ b/coin/provisioning/common/windows/mqtt_broker.ps1 @@ -43,7 +43,7 @@ Download $externalUrl $internalUrl $zip Verify-Checksum $zip $sha1 Write-Host "MQTT: Installing $zip..." -Extract-Zip $zip C:\Utils +Extract-7Zip $zip C:\Utils Remove-Item -Path $zip Set-EnvironmentVariable "MQTT_TEST_BROKER_LOCATION" "C:\Utils\paho.mqtt.testing-c342c09dadc7a664d0a8befad1ca031f5a0b0bc0\interoperability\startbroker.py" diff --git a/coin/provisioning/common/windows/mysql.ps1 b/coin/provisioning/common/windows/mysql.ps1 index a22e439a..29ce0ab4 100644 --- a/coin/provisioning/common/windows/mysql.ps1 +++ b/coin/provisioning/common/windows/mysql.ps1 @@ -37,8 +37,11 @@ # Both x86 and x64 versions needed when x86 integrations are done on x64 machine $version = "5.6.11" -$packagex64 = "C:\Windows\temp\mysql-$version-winx64.zip" -$packagex86 = "C:\Windows\temp\mysql-$version-win32.zip" +$baseNameX64 = "mysql-$version-winx64" +$packagex64 = "C:\Windows\temp\$baseNameX64.zip" +$baseNameX86 = "mysql-$version-win32" +$packagex86 = "C:\Windows\temp\$baseNameX86.zip" +$installFolder = "C:\Utils\my_sql" function DownloadAndInstall { @@ -52,7 +55,7 @@ function DownloadAndInstall Copy-Item $internalUrl $package $zipDir = [io.path]::GetFileNameWithoutExtension($package) - Extract-Dev-Folders-From-Zip $package $zipDir $installPath + Extract-7Zip $package $installPath "$zipDir\lib $zipDir\bin $zipDir\share $zipDir\include" Remove-Item -Path $package } @@ -60,28 +63,21 @@ function DownloadAndInstall if (Is64BitWinHost) { # Install x64 bit version $architecture = "x64" - $installFolder = "C:\Utils\my_sql\my_sql" $internalUrl = "\\ci-files01-hki.intra.qt.io\provisioning\windows\mysql-$version-winx64.zip" DownloadAndInstall $internalUrl $packagex64 $installFolder - Set-EnvironmentVariable "MYSQL_INCLUDE_x64" "$installFolder\include" - Set-EnvironmentVariable "MYSQL_LIB_x64" "$installFolder\lib" + Set-EnvironmentVariable "MYSQL_INCLUDE_x64" "$installFolder\$baseNameX64\include" + Set-EnvironmentVariable "MYSQL_LIB_x64" "$installFolder\$baseNameX64\lib" } # Install x86 bit version $architecture = "x86" $internalUrl = "\\ci-files01-hki.intra.qt.io\provisioning\windows\mysql-$version-win32.zip" -if (Is64BitWinHost) { - $installFolder = "C:\Utils\my_sql\my_sql$architecture" -} else { - $installFolder = "C:\Utils\my_sql\my_sql" -} - DownloadAndInstall $internalUrl $packagex86 $installFolder -Set-EnvironmentVariable "MYSQL_INCLUDE_x86" "$installFolder\include" -Set-EnvironmentVariable "MYSQL_LIB_x86" "$installFolder\lib" +Set-EnvironmentVariable "MYSQL_INCLUDE_x86" "$installFolder\$baseNameX86\include" +Set-EnvironmentVariable "MYSQL_LIB_x86" "$installFolder\$baseNameX86\lib" # Store version information to ~/versions.txt, which is used to print version information to provision log. Write-Output "MySQL = $version" >> ~/versions.txt diff --git a/coin/provisioning/common/windows/ninja.ps1 b/coin/provisioning/common/windows/ninja.ps1 index 1903501e..5b6729f5 100644 --- a/coin/provisioning/common/windows/ninja.ps1 +++ b/coin/provisioning/common/windows/ninja.ps1 @@ -5,7 +5,7 @@ $zip = "c:\users\qt\downloads\ninja-1.6.0-win-x86.zip" Download https://github.com/ninja-build/ninja/releases/download/v1.6.0/ninja-win.zip \\ci-files01-hki.intra.qt.io\provisioning\ninja\ninja-1.6.0-win-x86.zip $zip Verify-Checksum $zip "E01093F6533818425F8EFB0843CED7DCAABEA3B2" -Extract-Zip $zip C:\Utils\Ninja +Extract-7Zip $zip C:\Utils\Ninja Remove-Item -Path $zip Add-Path "C:\Utils\Ninja" diff --git a/coin/provisioning/common/windows/postgresql.ps1 b/coin/provisioning/common/windows/postgresql.ps1 index b9681809..46d7501a 100644 --- a/coin/provisioning/common/windows/postgresql.ps1 +++ b/coin/provisioning/common/windows/postgresql.ps1 @@ -43,7 +43,7 @@ $packagex86 = "C:\Windows\temp\postgresql-$version-windows-binaries.zip" if (Is64BitWinHost) { # Install x64 bit versions $architecture = "x64" - $installFolder = "C:\Utils\postgresql\pgsql" + $installFolder = "C:\Utils\postgresql" $externalUrl = "http://get.enterprisedb.com/postgresql/postgresql-$version-windows-x64-binaries.zip" $internalUrl = "\\ci-files01-hki.intra.qt.io\provisioning\windows\postgresql-$version-windows-x64-binaries.zip" $sha1 = "4da0453cdfda335e064d4437cf5bb9d356054cfd" @@ -52,13 +52,13 @@ if (Is64BitWinHost) { Download $externalUrl $internalUrl $packagex64 Verify-Checksum $packagex64 $sha1 Write-Host "Installing $packagex64 ..." - Extract-Dev-Folders-From-Zip $packagex64 "pgsql" $installFolder + Extract-7Zip $packagex64 $installFolder "pgsql\lib pgsql\bin pgsql\share pgsql\include" Write-Host "Remove downloaded $packagex64 ..." Remove-Item -Path $packagex64 - Set-EnvironmentVariable "POSTGRESQL_INCLUDE_x64" "$installFolder\include" - Set-EnvironmentVariable "POSTGRESQL_LIB_x64" "$installFolder\lib" + Set-EnvironmentVariable "POSTGRESQL_INCLUDE_x64" "$installFolder\pgsql\include" + Set-EnvironmentVariable "POSTGRESQL_LIB_x64" "$installFolder\pgsql\lib" } # Install x86 bit version @@ -67,9 +67,9 @@ $externalUrl = "http://get.enterprisedb.com/postgresql/postgresql-$version-windo $internalUrl = "\\ci-files01-hki.intra.qt.io\provisioning\windows\postgresql-$version-windows-binaries.zip" $sha1 = "eb4f01845e1592800edbb74f60944b6c0aca51a9" if (Is64BitWinHost) { - $installFolder = "C:\Utils\postgresql$architecture\pgsql" + $installFolder = "C:\Utils\postgresql$architecture" } else { - $installFolder = "C:\Utils\postgresql\pgsql" + $installFolder = "C:\Utils\postgresql" } @@ -77,13 +77,13 @@ Write-Host "Fetching from URL..." Download $externalUrl $internalUrl $packagex86 Verify-Checksum $packagex86 $sha1 Write-Host "Installing $packagex86 ..." -Extract-Dev-Folders-From-Zip $packagex86 "pgsql" $installFolder +Extract-7Zip $packagex86 $installFolder "pgsql\lib pgsql\bin pgsql\share pgsql\include" Write-Host "Remove downloaded $packagex86 ..." Remove-Item -Path $packagex86 -Set-EnvironmentVariable "POSTGRESQL_INCLUDE_x86" "$installFolder\include" -Set-EnvironmentVariable "POSTGRESQL_LIB_x86" "$installFolder\lib" +Set-EnvironmentVariable "POSTGRESQL_INCLUDE_x86" "$installFolder\pgsql\include" +Set-EnvironmentVariable "POSTGRESQL_LIB_x86" "$installFolder\pgsql\lib" # Store version information to ~/versions.txt, which is used to print version information to provision log. Write-Output "PostgreSQL = $version" >> ~/versions.txt diff --git a/coin/provisioning/common/windows/winrtrunner.ps1 b/coin/provisioning/common/windows/winrtrunner.ps1 index 89a64344..9cdd04ab 100644 --- a/coin/provisioning/common/windows/winrtrunner.ps1 +++ b/coin/provisioning/common/windows/winrtrunner.ps1 @@ -5,7 +5,7 @@ $url = "http://download.qt.io/development_releases/prebuilt/winrtrunner/winrtrun Download $url $url $zip Verify-Checksum $zip "C19098A4C9DBD20EDEB4E5E0D3E6A5BBBCA73C42" -Extract-Zip $zip C:\Utils\winrtrunner +Extract-7Zip $zip C:\Utils\winrtrunner Remove-Item -Path $zip Set-EnvironmentVariable "CI_WINRTRUNNER_PATH" "C:\Utils\winrtrunner" diff --git a/coin/provisioning/qtci-windows-10-x86_64/90-signing-tools.ps1 b/coin/provisioning/qtci-windows-10-x86_64/90-signing-tools.ps1 index 854bee01..daaab36f 100644 --- a/coin/provisioning/qtci-windows-10-x86_64/90-signing-tools.ps1 +++ b/coin/provisioning/qtci-windows-10-x86_64/90-signing-tools.ps1 @@ -40,5 +40,5 @@ $url = "http://ci-files01-hki.intra.qt.io/input/semisecure/sign/sign.zip" $destination = "C:\Windows\temp\sign.zip" Download $url $url $destination -Extract-Zip "$destination" "C:\Utils" +Extract-7Zip "$destination" "C:\Utils" Remove-Item -Path "$destination" -- cgit v1.2.3