Add script to build a lib against Qt packagev5.14.0-beta2-packagingv5.13.2-packaging

The script downloads qtbase build artifact and compiles
the given source checkout against it.

The build is archived and uploaded to remote disk.

Task-number: QTQAINFRA-3227
Change-Id: I71fa40703c1333b47c9b40c25b096e7434c801fc
Reviewed-by: Jani Heikkinen <jani.heikkinen@qt.io>

3 files changed, 258 insertions, 3 deletions

diff --git a/.gitignore b/.gitignore
index 1ded96a18..9773647f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,6 @@ Makefile
 /packaging-tools/qt-bld-dynamic
 /packaging-tools/qt-install-dynamic
 /packaging-tools/ifw-pkg
+/packaging-tools/qt_pkg
+/packaging-tools/lm_bld
+/packaging-tools/lm_install_root
diff --git a/packaging-tools/bld_lib.py b/packaging-tools/bld_lib.py
new file mode 100644
index 000000000..75af490b5
--- /dev/null
+++ b/packaging-tools/bld_lib.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#############################################################################
+##
+## Copyright (C) 2019 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the release tools of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+import os
+import sys
+import argparse
+import logging
+import urllib.request
+import tarfile
+import platform
+import shutil
+import glob
+import subprocess
+import re
+from time import gmtime, strftime
+from typing import List, Tuple
+from urllib.parse import urlparse
+from shutil import which
+from remote_uploader import RemoteUploader
+
+LOG_FMT_CI = "%(asctime)s %(levelname)s:%(filename)s:%(lineno)d(%(process)d): %(message)s"
+log = logging.getLogger("Bld")
+log.setLevel(logging.INFO)
+# Unify format of all messages
+try:
+    from rainbow_logging_handler import RainbowLoggingHandler
+    handler = RainbowLoggingHandler(sys.stderr, color_asctime=(None, None, False))
+except ImportError:
+    handler = logging.StreamHandler()
+
+formatter = logging.Formatter(LOG_FMT_CI)
+handler.setFormatter(formatter)
+log.addHandler(handler)
+
+
+def findFile(searchPath: str, fileName: str) -> str:
+    for root, dirs, files in os.walk(searchPath):
+        if fileName in files:
+            return os.path.join(root, fileName)
+    assert False, "Unbale to find: {0} from: {1}".format(fileName, searchPath)
+
+
+def collectLibs(searchPath: str) -> List[str]:
+    for root, dirs, files in os.walk(searchPath):
+        for dir in dirs:
+            if dir == "lib":
+                return [os.path.join(root, dir, x) for x in os.listdir(os.path.join(root, dir))]
+    assert False, "Unbale to find: 'lib' from: {0}".format(searchPath)
+
+def parseQtVersion(downloadUrlPath: str) -> str:
+    regex = re.compile(r'([\d.]+)')
+    for item in downloadUrlPath.split("/"):
+        m = regex.search(item)
+        if m:
+            return m.groups()[0]
+    assert False, "Could not parse Qt version number from: {0}".format(downloadUrlPath)
+
+
+def downloadQtPkg(args: argparse.Namespace, currentDir: str) -> Tuple[str, str]:
+    urlRes = urlparse(args.qtpkg)
+    assert urlRes.scheme and urlRes.netloc and urlRes.path, "Invalid URL: {0}".format(args.qtpkg)
+    qtVersion = parseQtVersion(urlRes.path)
+
+    saveAs = os.path.join(currentDir, os.path.basename(urlRes.path))
+    if os.path.exists(saveAs):
+        log.info("Using existing: %s", saveAs)
+    else:
+        log.info("Downloading: %s into: %s", args.qtpkg, saveAs)
+        urllib.request.urlretrieve(args.qtpkg, saveAs)
+
+    return saveAs, qtVersion
+
+
+def extractArchive(saveAs: str, currentDir: str) -> str:
+    qtDestDir = os.path.join(currentDir, "qt_pkg")
+    if not os.path.exists(qtDestDir):
+        os.makedirs(qtDestDir)
+        log.info("Extracting to: %s", qtDestDir)
+        if saveAs.endswith("tar.gz"):
+            with tarfile.open(saveAs, "r:gz") as tar:
+                tar.extractall(qtDestDir)
+        elif saveAs.endswith(".7z"):
+            try:
+                os.chdir(qtDestDir)
+                subprocess.check_call(['7z', 'x', saveAs])
+            except Exception as e:
+                log.error("Extracting 7z file failed: %s", str(e))
+                raise
+            finally:
+                os.chdir(currentDir)
+    return qtDestDir
+
+
+def build(qtDestDir: str, currentDir: str) -> str:
+    if platform.system().lower() == "windows":
+        qmakeToolName = "qmake.exe"
+        makeToolName = "nmake"
+    else:
+        qmakeToolName = "qmake"
+        makeToolName = "make"
+
+    qmakeTool = findFile(qtDestDir, qmakeToolName)
+    assert qmakeTool, "Could not find: {0} from: {1}".format(qmakeToolName, qtDestDir)
+
+    # patch
+    with open(os.path.join(os.path.dirname(qmakeTool), "qt.conf"), "w+") as f:
+        f.write("[Paths]\n")
+        f.write("Prefix=..\n")
+
+    proFile = glob.glob(os.path.join(args.src_path, "*.pro"))
+    assert proFile, "Could not find .pro file(s) from: {0}".format(args.src_path)
+    proFile = proFile[0]
+    log.info("Using .pro file: %s", proFile)
+
+    installRootDir = os.path.join(currentDir, "lib_install_root")
+    shutil.rmtree(installRootDir, ignore_errors=True)
+    os.makedirs(installRootDir)
+
+    bldDir = os.path.join(currentDir, "lib_bld")
+    shutil.rmtree(bldDir, ignore_errors=True)  # ignore if path did not exist
+    os.makedirs(bldDir)
+
+    try:
+        os.chdir(bldDir)
+        subprocess.check_call([qmakeTool, proFile])
+        subprocess.check_call([makeToolName])
+        # on windows chhop out the drive letter (e.g. 'C:'"
+        installRoot = installRootDir[2:] if platform.system().lower() == "windows" else installRootDir
+        subprocess.check_call([makeToolName, 'install', 'INSTALL_ROOT=' + installRoot])
+    except subprocess.CalledProcessError as buildError:
+        log.error("Failed to build the project: %s", str(buildError))
+        raise
+    except Exception as e:
+        log.error("Something bad happened: %s", str(e))
+        raise
+    finally:
+        os.chdir(currentDir)
+
+    return installRootDir
+
+
+def archive(args: argparse.Namespace, installRootDir: str, currentDir: str) -> str:
+    # strip out drive letter on Windows e.g. 'C:'
+    srcPath = args.src_path[2:] if platform.system().lower() == "windows" else args.src_path
+    archivePath = os.path.join(installRootDir, srcPath.lstrip(os.path.sep))
+    log.info("Archiving from: %s", archivePath)
+
+    libs = collectLibs(installRootDir)
+    for lib in libs:
+        shutil.copy2(lib, archivePath)
+
+    plat = platform.system().lower()
+    arch = "x86_64" if sys.maxsize > 2**32 else "x86"
+    artifactsFileName = "artifacts-" + plat + "-" + arch + ".7z"
+    artifactsFilePath = os.path.join(currentDir, artifactsFileName)
+    try:
+        os.chdir(archivePath)
+        subprocess.check_call(['7z', 'a', '-m0=lzma2', '-mmt=16', artifactsFilePath, '*'])
+    except Exception as e:
+        print(e)
+        raise
+    finally:
+        os.chdir(currentDir)
+
+    log.info("Created artifact: %s", artifactsFilePath)
+    return artifactsFilePath
+
+
+def handleBuild(args: argparse.Namespace) -> None:
+    currentDir = os.getcwd()
+
+    saveAs, qtVersion = downloadQtPkg(args, currentDir)
+    qtDestDir = extractArchive(saveAs, currentDir)
+    installRootDir = build(qtDestDir, currentDir)
+    artifactsFilePath = archive(args, installRootDir, currentDir)
+
+    remoteUploader = RemoteUploader(False, args.remote_server, args.username, args.remote_base_path, qtVersion, args.project_name)
+    remoteUploader.initRemoteSnapshotDir(args.build_id)
+    remoteUploader.copyToRemote(artifactsFilePath, "")
+    remoteUploader.updateLatestSymlink()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="Helper script to build a lib against qtbase artifact.")
+    parser.add_argument("--qtpkg", dest="qtpkg", type=str, default=os.getenv("QT_PKG_URL"), help="URL pointing to pre-built Qt bin package.")
+    parser.add_argument("--src-path", dest="src_path", type=str, default=os.getenv("SRC_PATH"), help="Path to sources")
+    parser.add_argument("--remote-server", dest="remote_server", type=str, default=os.getenv("PACKAGE_STORAGE_SERVER"), help="Output server for build artifacts")
+    parser.add_argument("--username", dest="username", type=str, default=os.getenv("PACKAGE_STORAGE_SERVER_USER"), help="Username for the output server")
+    parser.add_argument("--remote-base-path", dest="remote_base_path", type=str, default=os.getenv("PACKAGE_STORAGE_SERVER_BASE_DIR"), help="Base path for output")
+    parser.add_argument("--project-name", dest="project_name", type=str, default=os.getenv("PROJECT_NAME"), help="Base path for output")
+    parser.add_argument("--build-id", dest="build_id", type=str, default=strftime('%Y%m%d%H%M%S', gmtime()), help="Base path for output")
+    args = parser.parse_args(sys.argv[1:])
+
+    assert args.qtpkg, "You must define '--qtpkg'!"
+    assert args.src_path, "You must define '--src-path'!"
+    assert args.remote_server, "You must define '--remote-server'!"
+    assert args.username, "You must define '--username'!"
+    assert args.remote_base_path, "You must define '--remote-base-path'!"
+    assert args.project_name, "You must define '--project-name'!"
+    if not which("7z"):
+        log.error("Could not find '7z' from the system. This tool is needed for notarization. Aborting..")
+        sys.exit(1)
+
+    handleBuild(args)
diff --git a/packaging-tools/remote_uploader.py b/packaging-tools/remote_uploader.py
index ec305078a..338f31265 100755
--- a/packaging-tools/remote_uploader.py
+++ b/packaging-tools/remote_uploader.py
@@ -27,7 +27,16 @@
 ##
 #############################################################################
 
-import sh
+import platform
+try:
+    import sh
+except ImportError:
+    # fallback: emulate the sh API with pbs
+    import pbs
+    class Sh(object):
+        def __getattr__(self, attr):
+            return pbs.Command(attr)
+    sh = Sh()
 
 
 class RemoteUploader:
@@ -38,6 +47,7 @@ class RemoteUploader:
         self.ssh = sh.ssh.bake("-o", "GSSAPIAuthentication=no", "-o", "StrictHostKeyChecking=no", remoteServerUserName + '@' + remoteServer)
         self.remoteLogin = remoteServerUserName + '@' + remoteServer
         self.remoteTargetBaseDir = remoteBasePath + '/' + projectName + '/' + branch + '/'
+        self.remoteLatestLink = self.remoteTargetBaseDir + 'latest'
         self.remoteTargetDir = ''
 
     def initRemoteSnapshotDir(self, buildId):
@@ -58,7 +68,17 @@ class RemoteUploader:
         if destDirName:
             remoteDestination = remoteDestination + '/' + destDirName + '/'
         print("Copying [{0}] to [{1}]".format(fileName, remoteDestination))
-        self.remoteCopy = sh.rsync.bake(fileName, remoteDestination)
+        copyTool = sh.scp.bake if platform.system().lower() == "windows" else sh.rsync.bake
+        remoteCopy = copyTool(fileName, remoteDestination)
         if self.dryRun:
             return
-        self.remoteCopy()
+        remoteCopy()
+
+    def updateLatestSymlink(self, forceUpdate=True):
+        print("Creating remote symlink: [{0}] -> [{1}]".format(self.remoteLatestLink, self.remoteTargetDir))
+        if not self.dryRun:
+            options = "-sfn" if forceUpdate else "-sn"
+            try:
+                self.ssh.ln(options, self.remoteTargetDir, self.remoteLatestLink)
+            except sh.ErrorReturnCode_1:
+                print("Symbolic link already exists.")