Merge remote-tracking branch 'origin/5.15' into dev

Change-Id: I46f5d2dc758d0e1f23377c91ba7496793461771e

4 files changed, 258 insertions, 111 deletions

diff --git a/build_scripts/main.py b/build_scripts/main.py
index 8429c1fdb..84628d8e0 100644
--- a/build_scripts/main.py
+++ b/build_scripts/main.py
@@ -48,7 +48,7 @@ from textwrap import dedent
 import time
 from .config import config
 from .utils import get_python_dict
-from .options import OPTION
+from .options import DistUtilsCommandMixin, OPTION
 from .wheel_utils import (get_package_version, get_qt_version,
                           get_package_timestamp, macos_plat_name,
                           macos_pyside_min_deployment_target)
@@ -284,57 +284,6 @@ def check_allowed_python_version():
 
 qt_src_dir = ''
 
-if OPTION["QT_VERSION"] is None:
-    OPTION["QT_VERSION"] = "5"
-if OPTION["QMAKE"] is None:
-    OPTION["QMAKE"] = find_executable("qmake-qt5")
-if OPTION["QMAKE"] is None:
-    OPTION["QMAKE"] = find_executable("qmake")
-
-# make qtinfo.py independent of relative paths.
-if OPTION["QMAKE"] is not None and os.path.exists(OPTION["QMAKE"]):
-    OPTION["QMAKE"] = os.path.abspath(OPTION["QMAKE"])
-if OPTION["CMAKE"] is not None and os.path.exists(OPTION["CMAKE"]):
-    OPTION["CMAKE"] = os.path.abspath(OPTION["CMAKE"])
-
-if len(OPTION["QMAKE"]) == 0:
-    print("qmake could not be found.")
-    sys.exit(1)
-if not os.path.exists(OPTION["QMAKE"]):
-    print("'{}' does not exist.".format(OPTION["QMAKE"]))
-    sys.exit(1)
-
-if OPTION["CMAKE"] is None:
-    OPTION["CMAKE"] = find_executable("cmake")
-
-if OPTION["CMAKE"] is None:
-    print("cmake could not be found.")
-    sys.exit(1)
-if not os.path.exists(OPTION["CMAKE"]):
-    print("'{}' does not exist.".format(OPTION["CMAKE"]))
-    sys.exit(1)
-
-# First element is default
-available_mkspecs = ["msvc", "mingw", "ninja"] if sys.platform == "win32" else ["make", "ninja"]
-
-if OPTION["MAKESPEC"] is None:
-    OPTION["MAKESPEC"] = available_mkspecs[0]
-
-if OPTION["MAKESPEC"] not in available_mkspecs:
-    print('Invalid option --make-spec "{}". Available values are {}'.format(OPTION["MAKESPEC"],
-                                                                            available_mkspecs))
-    sys.exit(1)
-
-if OPTION["JOBS"]:
-    if sys.platform == 'win32' and OPTION["NO_JOM"]:
-        print("Option --jobs can only be used with jom on Windows.")
-        sys.exit(1)
-    else:
-        if not OPTION["JOBS"].startswith('-j'):
-            OPTION["JOBS"] = '-j' + OPTION["JOBS"]
-else:
-    OPTION["JOBS"] = ''
-
 
 def is_debug_python():
     return getattr(sys, "gettotalrefcount", None) is not None
@@ -357,10 +306,41 @@ def prefix():
     return name
 
 
-# Single global instance of QtInfo to be used later in multiple code
-# paths.
-qtinfo = QtInfo()
-qtinfo.setup(OPTION["QMAKE"], OPTION["QT_VERSION"])
+# Initialize, pull and checkout submodules
+def prepare_sub_modules():
+    print("Initializing submodules for PySide2 version: {}".format(
+        get_package_version()))
+    submodules_dir = os.path.join(setup_script_dir, "sources")
+
+    # Create list of [name, desired branch, absolute path, desired
+    # branch] and determine whether all submodules are present
+    need_init_sub_modules = False
+
+    for m in submodules:
+        module_name = m[0]
+        module_dir = m[1] if len(m) > 1 else ''
+        module_dir = os.path.join(submodules_dir, module_dir, module_name)
+        # Check for non-empty directory (repository checked out)
+        if not os.listdir(module_dir):
+            need_init_sub_modules = True
+            break
+
+    if need_init_sub_modules:
+        git_update_cmd = ["git", "submodule", "update", "--init"]
+        if run_process(git_update_cmd) != 0:
+            m = "Failed to initialize the git submodules: update --init failed"
+            raise DistutilsSetupError(m)
+        git_pull_cmd = ["git", "submodule", "foreach", "git", "fetch", "--all"]
+        if run_process(git_pull_cmd) != 0:
+            m = "Failed to initialize the git submodules: git fetch --all failed"
+            raise DistutilsSetupError(m)
+    else:
+        print("All submodules present.")
+
+    git_update_cmd = ["git", "submodule", "update"]
+    if run_process(git_update_cmd) != 0:
+        m = "Failed to checkout the correct git submodules SHA1s."
+        raise DistutilsSetupError(m)
 
 
 def prepare_build():
@@ -377,7 +357,7 @@ def prepare_build():
 
     # locate Qt sources for the documentation
     if OPTION["QT_SRC"] is None:
-        install_prefix = qtinfo.prefix_dir
+        install_prefix = QtInfo().prefix_dir
         if install_prefix:
             global qt_src_dir
             # In-source, developer build
@@ -387,9 +367,13 @@ def prepare_build():
                 qt_src_dir = os.path.join(os.path.dirname(install_prefix), 'Src', 'qtbase')
 
 
-class PysideInstall(_install):
+class PysideInstall(_install, DistUtilsCommandMixin):
+
+    user_options = _install.user_options + DistUtilsCommandMixin.mixin_user_options
+
     def __init__(self, *args, **kwargs):
         _install.__init__(self, *args, **kwargs)
+        DistUtilsCommandMixin.__init__(self)
 
     def initialize_options(self):
         _install.initialize_options(self)
@@ -408,6 +392,10 @@ class PysideInstall(_install):
             # similar cases.
             self.warn_dir = False
 
+    def finalize_options(self):
+        DistUtilsCommandMixin.mixin_finalize_options(self)
+        _install.finalize_options(self)
+
     def run(self):
         _install.run(self)
         print('--- Install completed ({}s)'.format(elapsed()))
@@ -471,13 +459,17 @@ class PysideInstallLib(_install_lib):
         return outfiles
 
 
-class PysideBuild(_build):
+class PysideBuild(_build, DistUtilsCommandMixin):
+
+    user_options = _build.user_options + DistUtilsCommandMixin.mixin_user_options
 
     def __init__(self, *args, **kwargs):
         _build.__init__(self, *args, **kwargs)
+        DistUtilsCommandMixin.__init__(self)
 
     def finalize_options(self):
         os_name_backup = os.name
+        DistUtilsCommandMixin.mixin_finalize_options(self)
         if sys.platform == 'darwin':
             self.plat_name = macos_plat_name()
             # This is a hack to circumvent the dubious check in
@@ -498,7 +490,6 @@ class PysideBuild(_build):
         _build.initialize_options(self)
         self.make_path = None
         self.make_generator = None
-        self.debug = False
         self.script_dir = None
         self.sources_dir = None
         self.build_dir = None
@@ -543,7 +534,7 @@ class PysideBuild(_build):
             py_scripts_dir = os.path.join(py_prefix, "bin")
         self.py_scripts_dir = py_scripts_dir
 
-        self.qtinfo = qtinfo
+        self.qtinfo = QtInfo()
         qt_dir = os.path.dirname(OPTION["QMAKE"])
         qt_version = get_qt_version()
 
@@ -583,7 +574,6 @@ class PysideBuild(_build):
 
         self.make_path = make_path
         self.make_generator = make_generator
-        self.debug = OPTION["DEBUG"]
         self.script_dir = script_dir
         self.st_build_dir = os.path.join(self.script_dir, self.build_lib)
         self.sources_dir = sources_dir
diff --git a/build_scripts/options.py b/build_scripts/options.py
index db2a7e367..eb5d438a0 100644
--- a/build_scripts/options.py
+++ b/build_scripts/options.py
@@ -38,10 +38,27 @@
 #############################################################################
 
 from __future__ import print_function
+import distutils.log as log
+from distutils.spawn import find_executable
 import sys
 import os
 import warnings
 
+from .qtinfo import QtInfo
+
+
+_AVAILABLE_MKSPECS = ["msvc", "mingw", "ninja"] if sys.platform == "win32" else ["make", "ninja"]
+
+
+# Global options not which are not part of the commands
+ADDITIONAL_OPTIONS = """
+Additional options:
+  --limited-api                        Use Limited API [yes/no]
+  ---macos-use-libc++                  Use libc++ on macOS
+  --snapshot-build                     Snapshot build
+  --package-timestamp                  Package Timestamp
+"""
+
 
 def _warn_multiple_option(option):
     warnings.warn('Option "{}" occurs multiple times on the command line.'.format(option))
@@ -132,60 +149,192 @@ def option_value(*args, **kwargs):
     return options.option_value(*args, **kwargs)
 
 
-# Declare options
+def _jobs_option_value():
+    """Option value for parallel builds."""
+    value = option_value('parallel', short_option_name='j')
+    if value:
+        return '-j' + value if not value.startswith('-j') else value
+    return ''
+
+
+# Declare options which need to be known when instantiating the DistUtils
+# commands.
 OPTION = {
     "BUILD_TYPE": option_value("build-type"),
     "INTERNAL_BUILD_TYPE": option_value("internal-build-type"),
-    "DEBUG": has_option("debug"),
-    "RELWITHDEBINFO": has_option('relwithdebinfo'),
-    "QMAKE": option_value("qmake"),
-    "QT_VERSION": option_value("qt"),
-    "CMAKE": option_value("cmake"),
-    "OPENSSL": option_value("openssl"),
-    "SHIBOKEN_CONFIG_DIR": option_value("shiboken-config-dir"),
-    "ONLYPACKAGE": has_option("only-package"),
-    "STANDALONE": has_option("standalone"),
-    "MAKESPEC": option_value("make-spec"),
-    "IGNOREGIT": has_option("ignore-git"),
-    # don't generate documentation
-    "SKIP_DOCS": has_option("skip-docs"),
-    # don't include pyside2-examples
-    "NOEXAMPLES": has_option("no-examples"),
     # number of parallel build jobs
-    "JOBS": option_value('parallel', short_option_name='j'),
+    "JOBS": _jobs_option_value(),
     # Legacy, not used any more.
     "JOM": has_option('jom'),
-    # Do not use jom instead of nmake with msvc
-    "NO_JOM": has_option('no-jom'),
-    "BUILDTESTS": has_option("build-tests"),
-    "MACOS_ARCH": option_value("macos-arch"),
     "MACOS_USE_LIBCPP": has_option("macos-use-libc++"),
-    "MACOS_SYSROOT": option_value("macos-sysroot"),
-    "MACOS_DEPLOYMENT_TARGET": option_value("macos-deployment-target"),
-    "XVFB": has_option("use-xvfb"),
-    "REUSE_BUILD": has_option("reuse-build"),
-    "SKIP_CMAKE": has_option("skip-cmake"),
-    "SKIP_MAKE_INSTALL": has_option("skip-make-install"),
-    "SKIP_PACKAGING": has_option("skip-packaging"),
-    "SKIP_MODULES": option_value("skip-modules"),
-    "MODULE_SUBSET": option_value("module-subset"),
-    "RPATH_VALUES": option_value("rpath"),
-    "QT_CONF_PREFIX": option_value("qt-conf-prefix"),
-    "QT_SRC": option_value("qt-src-dir"),
     "QUIET": has_option('quiet', remove=False),
-    "VERBOSE_BUILD": has_option("verbose-build"),
-    "SANITIZE_ADDRESS": has_option("sanitize-address"),
     "SNAPSHOT_BUILD": has_option("snapshot-build"),
     "LIMITED_API": option_value("limited-api"),
     "PACKAGE_TIMESTAMP": option_value("package-timestamp"),
-    "SHORTER_PATHS": has_option("shorter-paths"),
     # This is used automatically by distutils.command.install object, to
     # specify the final installation location.
-    "FINAL_INSTALL_PREFIX": option_value("prefix", remove=False),
+    "FINAL_INSTALL_PREFIX": option_value("prefix", remove=False)
     # This is used to identify the template for doc builds
-    "DOC_BUILD_ONLINE": has_option("doc-build-online"),
 }
 _deprecated_option_jobs = option_value('jobs')
 if _deprecated_option_jobs:
     _warn_deprecated_option('jobs', 'parallel')
     OPTION["JOBS"] = _deprecated_option_jobs
+
+
+class DistUtilsCommandMixin(object):
+    """Mixin for the DistUtils build/install commands handling the options."""
+
+    _finalized = False
+
+    mixin_user_options = [
+        ('debug', None, 'Build with debug information'),
+        ('relwithdebinfo', None, 'Build in release mode with debug information'),
+        ('only-package', None, 'Package only'),
+        ('standalone', None, 'Standalone build'),
+        ('ignore-git', None, 'Do update subrepositories'),
+        ('skip-docs', None, 'Skip documentation build'),
+        ('no-examples', None, 'Do not build examples'),
+        ('no-jom', None, 'Do not use jom (MSVC)'),
+        ('build-tests', None, 'Build tests'),
+        ('use-xvfb', None, 'Use Xvfb for testing'),
+        ('reuse-build', None, 'Reuse existing build'),
+        ('skip-cmake', None, 'Skip CMake step'),
+        ('skip-make-install', None, 'Skip install step'),
+        ('skip-packaging', None, 'Skip packaging step'),
+        ('verbose-build', None, 'Verbose build'),
+        ('sanitize-address', None, 'Build with address sanitizer'),
+        ('shorter-paths', None, 'Use shorter paths'),
+        ('doc-build-online', None, 'Build online documentation'),
+        ('qmake=', None, 'Path to qmake'),
+        ('qt=', None, 'Qt version'),
+        ('cmake=', None, 'Path to CMake'),
+        ('openssl=', None, 'Path to OpenSSL libraries'),
+        ('shiboken-config-dir=', None, 'shiboken configuration directory'),
+        ('make-spec=', None, 'Qt make-spec'),
+        ('macos-arch=', None, 'macOS architecture'),
+        ('macos-sysroot=', None, 'macOS sysroot'),
+        ('macos-deployment-target=', None, 'macOS deployment target'),
+        ('skip-modules=', None, 'Qt modules to be skipped'),
+        ('module-subset=', None, 'Qt modules to be built'),
+        ('rpath=', None, 'RPATH'),
+        ('qt-conf-prefix=', None, 'Qt configuration prefix'),
+        ('qt-src-dir=', None, 'Qt source directory')]
+
+    def __init__(self):
+        self.debug = False
+        self.relwithdebinfo = False
+        self.only_package = False
+        self.standalone = False
+        self.ignore_git = False
+        self.skip_docs = False
+        self.no_examples = False
+        self.no_jom = False
+        self.build_tests = False
+        self.use_xvfb = False
+        self.reuse_build = False
+        self.skip_cmake = False
+        self.skip_make_install = False
+        self.skip_packaging = False
+        self.verbose_build = False
+        self.sanitize_address = False
+        self.snapshot_build = False
+        self.shorter_paths = False
+        self.doc_build_online = False
+        self.qmake = None
+        self.qt = '5'
+        self.cmake = None
+        self.openssl = None
+        self.shiboken_config_dir = None
+        self.make_spec = None
+        self.macos_arch = None
+        self.macos_sysroot = None
+        self.macos_deployment_target = None
+        self.skip_modules = None
+        self.module_subset = None
+        self.rpath = None
+        self.qt_conf_prefix = None
+        self.qt_src_dir = None
+
+    def mixin_finalize_options(self):
+        # Bail out on 2nd call to mixin_finalize_options() since that is the
+        # build command following the install command when invoking
+        # setup.py install
+        if not DistUtilsCommandMixin._finalized:
+            DistUtilsCommandMixin._finalized = True
+            self._do_finalize()
+
+    def _do_finalize(self):
+        if not self._determine_defaults_and_check():
+            sys.exit(-1)
+        OPTION['DEBUG'] = self.debug
+        OPTION['RELWITHDEBINFO'] = self.relwithdebinfo
+        OPTION['ONLYPACKAGE'] = self.only_package
+        OPTION['STANDALONE'] = self.standalone
+        OPTION['IGNOREGIT'] = self.ignore_git
+        OPTION['SKIP_DOCS'] = self.skip_docs
+        OPTION['NOEXAMPLES'] = self.no_examples
+        OPTION['BUILDTESTS'] = self.build_tests
+        OPTION['NO_JOM'] = self.no_jom
+        OPTION['XVFB'] = self.use_xvfb
+        OPTION['REUSE_BUILD'] = self.reuse_build
+        OPTION['SKIP_CMAKE'] = self.skip_cmake
+        OPTION['SKIP_MAKE_INSTALL'] = self.skip_make_install
+        OPTION['SKIP_PACKAGING'] = self.skip_packaging
+        OPTION['VERBOSE_BUILD'] = self.verbose_build
+        if self.verbose_build:
+            log.set_verbosity(1)
+        OPTION['SANITIZE_ADDRESS'] = self.sanitize_address
+        OPTION['SHORTER_PATHS'] = self.shorter_paths
+        OPTION['DOC_BUILD_ONLINE'] = self.doc_build_online
+        # make qtinfo.py independent of relative paths.
+        qmake_abs_path = os.path.abspath(self.qmake)
+        OPTION['QMAKE'] = qmake_abs_path
+        OPTION['QT_VERSION'] = self.qt
+        QtInfo().setup(qmake_abs_path, self.qt)
+        OPTION['CMAKE'] = os.path.abspath(self.cmake)
+        OPTION['OPENSSL'] = self.openssl
+        OPTION['SHIBOKEN_CONFIG_DIR'] = self.shiboken_config_dir
+        OPTION['MAKESPEC'] = self.make_spec
+        OPTION['MACOS_ARCH'] = self.macos_arch
+        OPTION['MACOS_SYSROOT'] = self.macos_sysroot
+        OPTION['MACOS_DEPLOYMENT_TARGET'] = self.macos_deployment_target
+        OPTION['SKIP_MODULES'] = self.skip_modules
+        OPTION['MODULE_SUBSET'] = self.module_subset
+        OPTION['RPATH_VALUES'] = self.rpath
+        OPTION['QT_CONF_PREFIX'] = self.qt_conf_prefix
+        OPTION['QT_SRC'] = self.qt_src_dir
+
+    def _determine_defaults_and_check(self):
+        if not self.cmake:
+            self.cmake = find_executable("cmake")
+        if not self.cmake:
+            print("cmake could not be found.")
+            return False
+        if not os.path.exists(self.cmake):
+            print("'{}' does not exist.".format(self.cmake))
+            return False
+
+        if not self.qmake:
+            self.qmake = find_executable("qmake")
+            if not self.qmake:
+                self.qmake = find_executable("qmake-qt5")
+        if not self.qmake:
+            print("qmake could not be found.")
+            return False
+        if not os.path.exists(self.qmake):
+            print("'{}' does not exist.".format(self.qmake))
+            return False
+
+        if not self.make_spec:
+            self.make_spec = _AVAILABLE_MKSPECS[0]
+        if self.make_spec not in _AVAILABLE_MKSPECS:
+            print('Invalid option --make-spec "{}". Available values are {}'.format(OPTION["MAKESPEC"],
+                                                                                    _AVAILABLE_MKSPECS))
+            return False
+
+        if OPTION["JOBS"] and sys.platform == 'win32' and self.no_jom:
+            print("Option --jobs can only be used with jom on Windows.")
+            return False
+
+        return True
diff --git a/build_scripts/setup_runner.py b/build_scripts/setup_runner.py
index b54c62796..6b944c2c8 100644
--- a/build_scripts/setup_runner.py
+++ b/build_scripts/setup_runner.py
@@ -46,13 +46,11 @@ import distutils.log as log
 from build_scripts.config import config
 from build_scripts.main import get_package_version, get_setuptools_extension_modules
 from build_scripts.main import cmd_class_dict
-from build_scripts.options import OPTION
+from build_scripts.options import ADDITIONAL_OPTIONS, OPTION
 from build_scripts.utils import run_process
 
 from setuptools import setup
 
-if OPTION["VERBOSE_BUILD"]:
-    log.set_verbosity(1)
 
 
 class SetupRunner(object):
@@ -166,6 +164,10 @@ class SetupRunner(object):
                     """).format(exit_code, cmd_as_string)
                 raise RuntimeError(msg)
 
+        if help_requested:
+            print(ADDITIONAL_OPTIONS)
+
+
     @staticmethod
     def run_setuptools_setup():
         """
diff --git a/build_scripts/wheel_override.py b/build_scripts/wheel_override.py
index e4147a5bc..20e6f942c 100644
--- a/build_scripts/wheel_override.py
+++ b/build_scripts/wheel_override.py
@@ -40,22 +40,22 @@
 
 wheel_module_exists = False
 
+import os
+import sys
+from .options import DistUtilsCommandMixin, OPTION
+from distutils import log as logger
+from email.generator import Generator
+from .wheel_utils import get_package_version, get_qt_version, macos_plat_name
+
 try:
-    import os
-    import sys
 
-    from distutils import log as logger
     from wheel import pep425tags
     from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
     from wheel.bdist_wheel import safer_name as _safer_name
     from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag
     from wheel.pep425tags import get_platform as wheel_get_platform
-    from email.generator import Generator
     from wheel import __version__ as wheel_version
 
-    from .options import OPTION
-    from .wheel_utils import get_package_version, get_qt_version, macos_plat_name
-
     wheel_module_exists = True
 except Exception as e:
     _bdist_wheel, wheel_version = type, ''  # dummy to make class statement happy
@@ -67,12 +67,18 @@ def get_bdist_wheel_override():
     return PysideBuildWheel if wheel_module_exists else None
 
 
-class PysideBuildWheel(_bdist_wheel):
+class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin):
+
+    user_options = (_bdist_wheel.user_options + DistUtilsCommandMixin.mixin_user_options
+                    if wheel_module_exists else None)
+
     def __init__(self, *args, **kwargs):
         self._package_version = None
         _bdist_wheel.__init__(self, *args, **kwargs)
+        DistUtilsCommandMixin.__init__(self)
 
     def finalize_options(self):
+        DistUtilsCommandMixin.mixin_finalize_options(self)
         if sys.platform == 'darwin':
             # Override the platform name to contain the correct
             # minimum deployment target.