Add tests to install built wheels and build some examples

Augment coin_test_instructions.py to run a new script
testing/wheel_tester.py, which tries to pip install the built
wheels, and then tries to build the samplebinding and
scriptableapplication examples.

This allows us to confirm that the generated wheels are actually
installable, and also hopefully prevent us from breaking the
embeddable examples, by making sure that they at least build
(and execute in the case of samplebinding).

The change also modifies the examples to be able to take the
python executable as build argument, so that wheel_tester
can specify explicitly which python interpeter to use.

Change-Id: I0f141e40ab86e3311dd25915c4e53b0af36aaae9
Reviewed-by: Christian Tismer <tismer@stackless.com>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

7 files changed, 349 insertions, 21 deletions

diff --git a/build_scripts/utils.py b/build_scripts/utils.py
index 165366e26..741de73b9 100644
--- a/build_scripts/utils.py
+++ b/build_scripts/utils.py
@@ -1141,3 +1141,13 @@ def acceptCITestConfiguration(hostOS, hostOSVer, targetArch, compiler):
         print("Disabled " + compiler + " to " + targetArch + " from Coin configuration")
         return False
     return True
+
+
+def get_ci_qmake_path(ci_install_dir, ci_host_os):
+    qmake_path = "--qmake={}".format(ci_install_dir)
+    if ci_host_os == "MacOS":
+        return qmake_path + "/bin/qmake"
+    elif ci_host_os == "Windows":
+        return qmake_path + "\\bin\\qmake.exe"
+    else:
+        return qmake_path + "/bin/qmake"
diff --git a/coin_build_instructions.py b/coin_build_instructions.py
index 6ef17246a..1104e996c 100644
--- a/coin_build_instructions.py
+++ b/coin_build_instructions.py
@@ -45,6 +45,7 @@ from build_scripts.utils import run_instruction
 from build_scripts.utils import rmtree
 from build_scripts.utils import get_python_dict
 from build_scripts.utils import acceptCITestConfiguration
+from build_scripts.utils import get_ci_qmake_path
 import os
 
 # Values must match COIN thrift
@@ -108,13 +109,8 @@ def call_setup(python_ver):
         cmd += ["bdist_wheel", "--standalone"]
     else:
         cmd += ["build"]
-    if CI_HOST_OS == "MacOS":
-        cmd += ["--qmake=" + CI_ENV_INSTALL_DIR + "/bin/qmake"]
-    elif CI_HOST_OS == "Windows":
-
-        cmd += ["--qmake=" + CI_ENV_INSTALL_DIR + "\\bin\\qmake.exe"]
-    else:
-        cmd += ["--qmake=" + CI_ENV_INSTALL_DIR + "/bin/qmake"]
+    qmake_path = get_ci_qmake_path(CI_ENV_INSTALL_DIR, CI_HOST_OS)
+    cmd.append(qmake_path)
     cmd += ["--build-tests",
             "--jobs=4",
             "--verbose-build"]
diff --git a/coin_test_instructions.py b/coin_test_instructions.py
index 5ecb4c17a..a61a1c9fa 100644
--- a/coin_test_instructions.py
+++ b/coin_test_instructions.py
@@ -44,6 +44,7 @@ from build_scripts.utils import get_qtci_virtualEnv
 from build_scripts.utils import run_instruction
 from build_scripts.utils import rmtree
 from build_scripts.utils import acceptCITestConfiguration
+from build_scripts.utils import get_ci_qmake_path
 import os
 
 # Values must match COIN thrift
@@ -74,6 +75,14 @@ def call_testrunner(python_ver, buildnro):
                   "--buildno=" + buildnro]
     run_instruction(cmd, "Failed to run testrunner.py")
 
+    qmake_path = get_ci_qmake_path(CI_ENV_INSTALL_DIR, CI_HOST_OS)
+
+    # Try to install built wheels, and build some buildable examples.
+    if CI_RELEASE_CONF:
+        wheel_tester_path = os.path.join("testing", "wheel_tester.py")
+        cmd = [env_python, wheel_tester_path, qmake_path]
+        run_instruction(cmd, "Error while running wheel_tester.py")
+
 def run_test_instructions():
     if not acceptCITestConfiguration(CI_HOST_OS, CI_HOST_OS_VER, CI_TARGET_ARCH, CI_COMPILER):
         exit()
diff --git a/examples/samplebinding/CMakeLists.txt b/examples/samplebinding/CMakeLists.txt
index f5212c449..3852ed36f 100644
--- a/examples/samplebinding/CMakeLists.txt
+++ b/examples/samplebinding/CMakeLists.txt
@@ -40,7 +40,11 @@ set(generated_sources
 
 
 # ================================== Shiboken detection ======================================
-
+# Use provided python interpreter if given.
+if(NOT python_interpreter)
+    find_program(python_interpreter "python")
+endif()
+message(STATUS "Using python interpreter: ${python_interpreter}")
 
 # Macro to get various pyside / python include / link flags and paths.
 # Uses the not entirely supported utils/pyside2_config.py file.
@@ -52,7 +56,8 @@ macro(pyside2_config option output_var)
     endif()
 
     execute_process(
-      COMMAND python "${CMAKE_SOURCE_DIR}/../utils/pyside2_config.py" ${option}
+      COMMAND ${python_interpreter} "${CMAKE_SOURCE_DIR}/../utils/pyside2_config.py"
+              ${option}
       OUTPUT_VARIABLE ${output_var}
       OUTPUT_STRIP_TRAILING_WHITESPACE)
 
diff --git a/examples/scriptableapplication/CMakeLists.txt b/examples/scriptableapplication/CMakeLists.txt
index 71e7869ab..215d08961 100644
--- a/examples/scriptableapplication/CMakeLists.txt
+++ b/examples/scriptableapplication/CMakeLists.txt
@@ -14,6 +14,12 @@ set(CMAKE_CXX_STANDARD 11)
 # Find required Qt packages.
 find_package(Qt5 5.9 REQUIRED COMPONENTS Core Gui Widgets)
 
+# Use provided python interpreter if given.
+if(NOT python_interpreter)
+    find_program(python_interpreter "python")
+endif()
+message(STATUS "Using python interpreter: ${python_interpreter}")
+
 # Macro to get various pyside / python include / link flags.
 macro(pyside2_config option output_var)
     if(${ARGC} GREATER 2)
@@ -23,7 +29,8 @@ macro(pyside2_config option output_var)
     endif()
 
     execute_process(
-      COMMAND python "${CMAKE_SOURCE_DIR}/../utils/pyside2_config.py" ${option}
+      COMMAND ${python_interpreter} "${CMAKE_SOURCE_DIR}/../utils/pyside2_config.py"
+              ${option}
       OUTPUT_VARIABLE ${output_var}
       OUTPUT_STRIP_TRAILING_WHITESPACE)
 
diff --git a/examples/scriptableapplication/pyside2.pri b/examples/scriptableapplication/pyside2.pri
index a2dc516cf..2da3bc880 100644
--- a/examples/scriptableapplication/pyside2.pri
+++ b/examples/scriptableapplication/pyside2.pri
@@ -1,36 +1,42 @@
 PYSIDE_CONFIG = $$PWD/../utils/pyside2_config.py
 
-SHIBOKEN2_GENERATOR = $$system(python $$PYSIDE_CONFIG --shiboken2-generator-path)
+# Use provided python interpreter if given.
+isEmpty(python_interpreter) {
+    python_interpreter = python
+}
+message(Using python interpreter: $$python_interpreter)
+
+SHIBOKEN2_GENERATOR = $$system($$python_interpreter $$PYSIDE_CONFIG --shiboken2-generator-path)
 isEmpty(SHIBOKEN2_GENERATOR): error(Unable to locate the shiboken2-generator package location)
 
-SHIBOKEN2_MODULE = $$system(python $$PYSIDE_CONFIG --shiboken2-module-path)
+SHIBOKEN2_MODULE = $$system($$python_interpreter $$PYSIDE_CONFIG --shiboken2-module-path)
 isEmpty(SHIBOKEN2_MODULE): error(Unable to locate the shiboken2 package location)
 
-PYSIDE2 = $$system(python $$PYSIDE_CONFIG --pyside2-path)
+PYSIDE2 = $$system($$python_interpreter $$PYSIDE_CONFIG --pyside2-path)
 isEmpty(PYSIDE2): error(Unable to locate the PySide2 package location)
 
-PYTHON_INCLUDE = $$system(python $$PYSIDE_CONFIG --python-include-path)
+PYTHON_INCLUDE = $$system($$python_interpreter $$PYSIDE_CONFIG --python-include-path)
 isEmpty(PYTHON_INCLUDE): error(Unable to locate the Python include headers directory)
 
-PYTHON_LFLAGS = $$system(python $$PYSIDE_CONFIG --python-link-flags-qmake)
+PYTHON_LFLAGS = $$system($$python_interpreter $$PYSIDE_CONFIG --python-link-flags-qmake)
 isEmpty(PYTHON_LFLAGS): error(Unable to locate the Python library for linking)
 
-SHIBOKEN2_INCLUDE = $$system(python $$PYSIDE_CONFIG --shiboken2-generator-include-path)
+SHIBOKEN2_INCLUDE = $$system($$python_interpreter $$PYSIDE_CONFIG --shiboken2-generator-include-path)
 isEmpty(SHIBOKEN2_INCLUDE): error(Unable to locate the shiboken include headers directory)
 
-PYSIDE2_INCLUDE = $$system(python $$PYSIDE_CONFIG --pyside2-include-path)
+PYSIDE2_INCLUDE = $$system($$python_interpreter $$PYSIDE_CONFIG --pyside2-include-path)
 isEmpty(PYSIDE2_INCLUDE): error(Unable to locate the PySide2 include headers directory)
 
-SHIBOKEN2_LFLAGS = $$system(python $$PYSIDE_CONFIG --shiboken2-module-qmake-lflags)
+SHIBOKEN2_LFLAGS = $$system($$python_interpreter $$PYSIDE_CONFIG --shiboken2-module-qmake-lflags)
 isEmpty(SHIBOKEN2_LFLAGS): error(Unable to locate the shiboken libraries for linking)
 
-PYSIDE2_LFLAGS = $$system(python $$PYSIDE_CONFIG --pyside2-qmake-lflags)
+PYSIDE2_LFLAGS = $$system($$python_interpreter $$PYSIDE_CONFIG --pyside2-qmake-lflags)
 isEmpty(PYSIDE2_LFLAGS): error(Unable to locate the PySide2 libraries for linking)
 
-SHIBOKEN2_SHARED_LIBRARIES = $$system(python $$PYSIDE_CONFIG --shiboken2-module-shared-libraries-qmake)
+SHIBOKEN2_SHARED_LIBRARIES = $$system($$python_interpreter $$PYSIDE_CONFIG --shiboken2-module-shared-libraries-qmake)
 isEmpty(SHIBOKEN2_SHARED_LIBRARIES): error(Unable to locate the used shiboken2 module shared libraries)
 
-PYSIDE2_SHARED_LIBRARIES = $$system(python $$PYSIDE_CONFIG --pyside2-shared-libraries-qmake)
+PYSIDE2_SHARED_LIBRARIES = $$system($$python_interpreter $$PYSIDE_CONFIG --pyside2-shared-libraries-qmake)
 isEmpty(PYSIDE2_SHARED_LIBRARIES): error(Unable to locate the used PySide2 shared libraries)
 
 INCLUDEPATH += "$$PYTHON_INCLUDE" $$PYSIDE2_INCLUDE $$SHIBOKEN2_INCLUDE
diff --git a/testing/wheel_tester.py b/testing/wheel_tester.py
new file mode 100644
index 000000000..60fd7a38a
--- /dev/null
+++ b/testing/wheel_tester.py
@@ -0,0 +1,295 @@
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+"""
+This script is used by Coin (coin_test_instructions.py specifically) to
+test installation of generated wheels, and test building of the
+"buildable" examples samplebinding and scriptableapplication.
+
+It can also be invoked regularly from the command line via
+python testing/wheel_tester.py --qmake=some-value --cmake=some-value
+
+The qmake and cmake arguments can also be omitted, and they will be
+looked up in your PATH.
+
+Make sure that some generated wheels already exist in the dist/
+directory (e.g. setup.py bdist_wheel was already executed).
+"""
+
+import os, sys
+
+try:
+    this_file = __file__
+except NameError:
+    this_file = sys.argv[0]
+this_file = os.path.abspath(this_file)
+this_dir = os.path.dirname(this_file)
+setup_script_dir = os.path.abspath(os.path.join(this_dir, '..'))
+sys.path.append(setup_script_dir)
+
+from build_scripts.options import OPTION_QMAKE
+from build_scripts.options import OPTION_CMAKE
+
+from build_scripts.utils import find_files_using_glob
+from build_scripts.utils import find_glob_in_path
+from build_scripts.utils import run_process
+from build_scripts.utils import rmtree
+import distutils.log as log
+
+log.set_verbosity(1)
+
+
+def find_executable_qmake():
+    return find_executable('qmake', OPTION_QMAKE)
+
+
+def find_executable_cmake():
+    return find_executable('cmake', OPTION_CMAKE)
+
+
+def find_executable(executable, command_line_value):
+    value = command_line_value
+    option_str = '--{}'.format(executable)
+
+    if value:
+        log.info("{} option given: {}".format(option_str, value))
+        if not os.path.exists(value):
+            raise RuntimeError("No executable exists at: {}".format(value))
+    else:
+        log.info("No {} option given, trying to find {} in PATH.".format(option_str, executable))
+        paths = find_glob_in_path(executable)
+        log.info("{} executables found in PATH: {}".format(executable, paths))
+        if not paths:
+            raise RuntimeError(
+                "No {} option was specified and no {} was found "
+                "in PATH.".format(option_str, executable))
+        else:
+            value = paths[0]
+            log.info("Using {} found in PATH: {}".format(executable, value))
+    log.info("")
+    return value
+
+
+QMAKE_PATH = find_executable_qmake()
+CMAKE_PATH = find_executable_cmake()
+
+
+def get_wheels_dir():
+    return os.path.join(setup_script_dir, "dist")
+
+
+def get_examples_dir():
+    return os.path.join(setup_script_dir, "examples")
+
+
+def package_prefix_names():
+    return ["shiboken2", "shiboken2_generator", "PySide2"]
+
+
+def clean_egg_info():
+    # After a successful bdist_wheel build, some .egg-info directories
+    # are left over, which confuse pip when invoking it via
+    # python -m pip, making pip think that the packages are already
+    # installed in the root source directory.
+    # Clean up the .egg-info directories to fix this, it should be
+    # safe to do so.
+    paths = find_files_using_glob(setup_script_dir, "*.egg-info")
+    for p in paths:
+        log.info("Removing {}".format(p))
+        rmtree(p)
+
+
+def install_wheel(wheel_path):
+    log.info("Installing wheel: {}".format(wheel_path))
+    exit_code = run_process([sys.executable, "-m", "pip", "install", wheel_path])
+    log.info("")
+    if exit_code:
+        raise RuntimeError("Error while installing wheel {}".format(wheel_path))
+
+
+def try_install_wheels(wheels_dir, py_version):
+    clean_egg_info()
+    all_wheels_pattern = "*.whl"
+    all_wheels = find_files_using_glob(wheels_dir, all_wheels_pattern)
+
+    if len(all_wheels) > 1:
+        log.info("Found the following wheels in {}: ".format(wheels_dir))
+        for wheel in all_wheels:
+            log.info(wheel)
+    else:
+        log.info("No wheels found in {}".format(wheels_dir))
+    log.info("")
+
+    for p in package_prefix_names():
+        pattern = "{}-*cp{}*.whl".format(p, py_version)
+        files = find_files_using_glob(wheels_dir, pattern)
+        if files and len(files) == 1:
+            wheel_path = files[0]
+            install_wheel(wheel_path)
+        elif len(files) > 1:
+            raise RuntimeError("More than one wheel found for specific package and version.")
+        else:
+            raise RuntimeError("No wheels compatible with Python {} found "
+                               "for testing.".format(py_version))
+
+
+def is_unix():
+    if sys.platform.startswith("linux") or sys.platform == "darwin":
+        return True
+    return False
+
+
+def generate_build_cmake():
+    args = [CMAKE_PATH]
+    if is_unix():
+        args.extend(["-G", "Unix Makefiles"])
+    else:
+        args.extend(["-G", "NMake Makefiles"])
+    args.append("-DCMAKE_BUILD_TYPE=Release")
+    args.append("-Dpython_interpreter={}".format(sys.executable))
+
+    # Specify prefix path so find_package(Qt5) works.
+    qmake_dir = os.path.abspath(os.path.join(os.path.dirname(QMAKE_PATH), ".."))
+    args.append("-DCMAKE_PREFIX_PATH={}".format(qmake_dir))
+
+    args.append("..")
+
+    exit_code = run_process(args)
+    if exit_code:
+        raise RuntimeError("Failure while running cmake.")
+    log.info("")
+
+
+def generate_build_qmake():
+    exit_code = run_process([QMAKE_PATH, "..", "python_interpreter={}".format(sys.executable)])
+    if exit_code:
+        raise RuntimeError("Failure while running qmake.")
+    log.info("")
+
+
+def run_make():
+    args = []
+    if is_unix():
+        executable = "make"
+    else:
+        executable = "nmake"
+    args.append(executable)
+
+    exit_code = run_process(args)
+    if exit_code:
+        raise RuntimeError("Failure while running {}.".format(executable))
+    log.info("")
+
+
+def run_make_install():
+    args = []
+    if is_unix():
+        executable = "make"
+    else:
+        executable = "nmake"
+    args.append(executable)
+    args.append("install")
+
+    exit_code = run_process(args)
+    if exit_code:
+        raise RuntimeError("Failed while running {} install.".format(executable))
+    log.info("")
+
+
+def execute_script(script_path):
+    args = [sys.executable, script_path]
+    exit_code = run_process(args)
+    if exit_code:
+        raise RuntimeError("Failure while executing script: {}".format(script_path))
+    log.info("")
+
+
+def prepare_build_folder(src_path, build_folder_name):
+    build_path = os.path.join(src_path, build_folder_name)
+
+    # The script can be called for both Python 2 and Python 3 wheels, so
+    # preparing a build folder should clean any previous existing build.
+    if os.path.exists(build_path):
+        log.info("Removing {}".format(build_path))
+        rmtree(build_path)
+
+    log.info("Creating {}".format(build_path))
+    os.makedirs(build_path)
+    os.chdir(build_path)
+
+
+def try_build_examples():
+    examples_dir = get_examples_dir()
+
+    log.info("Attempting to build and run samplebinding using cmake.")
+    src_path = os.path.join(examples_dir, "samplebinding")
+    prepare_build_folder(src_path, "cmake")
+    generate_build_cmake()
+    run_make()
+    run_make_install()
+    execute_script(os.path.join(src_path, "main.py"))
+
+    log.info("Attempting to build scriptableapplication using cmake.")
+    src_path = os.path.join(examples_dir, "scriptableapplication")
+    prepare_build_folder(src_path, "cmake")
+    generate_build_cmake()
+    run_make()
+
+    log.info("Attempting to build scriptableapplication using qmake.")
+    src_path = os.path.join(examples_dir, "scriptableapplication")
+    prepare_build_folder(src_path, "qmake")
+    generate_build_qmake()
+    run_make()
+
+
+def run_wheel_tests():
+    wheels_dir = get_wheels_dir()
+    py_version = sys.version_info[0]
+
+    log.info("Attempting to install wheels.\n")
+    try_install_wheels(wheels_dir, py_version)
+
+    log.info("Attempting to build examples.\n")
+    try_build_examples()
+
+    log.info("All tests passed!")
+
+
+if __name__ == "__main__":
+    run_wheel_tests()