path: root/coin_build_instructions.py

#############################################################################
##
## Copyright (C) 2018 The Qt Company Ltd.
## Contact: https://www.qt.io/licensing/
##
## This file is part of Qt for Python.
##
## $QT_BEGIN_LICENSE:LGPL$
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and The Qt Company. For licensing terms
## and conditions see https://www.qt.io/terms-conditions. For further
## information use the contact form at https://www.qt.io/contact-us.
##
## GNU Lesser General Public License Usage
## Alternatively, this file may be used under the terms of the GNU Lesser
## General Public License version 3 as published by the Free Software
## Foundation and appearing in the file LICENSE.LGPL3 included in the
## packaging of this file. Please review the following information to
## ensure the GNU Lesser General Public License version 3 requirements
## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
##
## GNU General Public License Usage
## Alternatively, this file may be used under the terms of the GNU
## General Public License version 2.0 or (at your option) the GNU General
## Public license version 3 or any later version approved by the KDE Free
## Qt Foundation. The licenses are as published by the Free Software
## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
## included in the packaging of this file. Please review the following
## information to ensure the GNU General Public License requirements will
## be met: https://www.gnu.org/licenses/gpl-2.0.html and
## https://www.gnu.org/licenses/gpl-3.0.html.
##
## $QT_END_LICENSE$
##
#############################################################################
import calendar
import datetime
import os
import os.path
import site
import sys
from os.path import expanduser
import pathlib
import urllib.request as urllib

from build_scripts.options import has_option, log, option_value
from build_scripts.utils import (expand_clang_variables, get_ci_qtpaths_path,
                                 get_qtci_virtualEnv,
                                 parse_cmake_conf_assignments_by_key, remove_tree,
                                 run_instruction)

log.set_verbosity(log.INFO)

# Values must match COIN thrift
CI_HOST_OS = option_value("os")
CI_TARGET_OS = option_value("targetOs")
CI_HOST_ARCH = option_value("hostArch")
CI_TARGET_ARCH = option_value("targetArch")
CI_HOST_OS_VER = option_value("osVer")
CI_ENV_INSTALL_DIR = option_value("instdir")
CI_ENV_AGENT_DIR = option_value("agentdir")
CI_COMPILER = option_value("compiler")
CI_USE_SCCACHE = option_value("compiler-launcher")
CI_INTEGRATION_ID = option_value("coinIntegrationId") or str(calendar.timegm(datetime.datetime.now().timetuple()))
CI_FEATURES = []
_ci_features = option_value("features")
if _ci_features is not None:
    for f in _ci_features.split(', '):
        CI_FEATURES.append(f)
CI_RELEASE_CONF  = has_option("packaging")
CI_TEST_PHASE = option_value("phase")
if CI_TEST_PHASE not in ["ALL", "BUILD", "WHEEL"]:
    CI_TEST_PHASE = "ALL"
CI_TEST_WITH_PYPY = has_option("pypy")

def get_current_script_path():
    """ Returns the absolute path containing this script. """
    try:
        this_file = __file__
    except NameError:
        this_file = sys.argv[0]
    this_file = os.path.abspath(this_file)
    return os.path.dirname(this_file)


def is_snapshot_build():
    """
    Returns True if project needs to be built with --snapshot-build

    This is true if the version found in .cmake.conf is not a
    pre-release version (no alphas, betas).

    This eliminates the need to remove the --snapshot-build option
    on a per-release branch basis (less things to remember to do
    for a release).
    """
    setup_script_dir = get_current_script_path()
    pyside_project_dir = os.path.join(setup_script_dir, "sources", "pyside6")

    d = parse_cmake_conf_assignments_by_key(pyside_project_dir)
    release_version_type = d.get('pyside_PRE_RELEASE_VERSION_TYPE')
    pre_release_version = d.get('pyside_PRE_RELEASE_VERSION')
    if pre_release_version and release_version_type:
        return True
    return False

def call_setup(python_ver, phase, pypy):
    print("call_setup")
    print("python_ver", python_ver)
    print("phase", phase)
    env_python = ""
    if python_ver == "pypy":
        print("running with " + pypy)
        env_python = pypy
    else:
        _pExe, _env, env_pip, env_python = get_qtci_virtualEnv(python_ver, CI_HOST_OS, CI_HOST_ARCH, CI_TARGET_ARCH)

        if phase in ["BUILD"]:
            remove_tree(_env, True)
            # Pinning the virtualenv before creating one
            # Use pip3 if possible while pip seems to install the virtualenv to wrong dir in some OS
            python3 = "python3"
            if sys.platform == "win32":
                python3 = os.path.join(os.getenv("PYTHON3_PATH"), "python.exe")
            run_instruction([python3, "-m", "pip", "install", "--user", "virtualenv==20.7.2"], "Failed to pin virtualenv")
            # installing to user base might not be in PATH by default.
            env_path = os.path.join(site.USER_BASE, "bin")
            v_env = os.path.join(env_path, "virtualenv")
            if sys.platform == "win32":
                env_path = os.path.join(site.USER_BASE, "Scripts")
                v_env = os.path.join(env_path, "virtualenv.exe")
            try:
                run_instruction([v_env, "--version"], "Using default virtualenv")
            except Exception as e:
                log.info("Failed to use the default virtualenv")
                log.info(f"{type(e).__name__}: {e}")
                v_env = "virtualenv"
            run_instruction([v_env, "-p", _pExe,  _env], "Failed to create virtualenv")
            # When the 'python_ver' variable is empty, we are using Python 2
            # Pip is always upgraded when CI template is provisioned, upgrading it in later phase may cause perm issue
            run_instruction([env_pip, "install", "-r", "requirements.txt"], "Failed to install dependencies")


    cmd = [env_python, "-u", "setup.py"]
    if phase in ["BUILD"]:
        cmd += ["build", "--standalone"]
    elif phase in ["WHEEL"] or CI_RELEASE_CONF:
        cmd += ["bdist_wheel", "--reuse-build", "--standalone", "--skip-cmake", "--skip-make-install", "--only-package"]

    cmd += ["--build-tests",
            "--verbose-build"]

    if CI_TARGET_ARCH == "X86_64-ARM64":
        cmd += ["--macos-arch='x86_64;arm64'"]

    if CI_USE_SCCACHE:
        cmd += [f"--compiler-launcher={CI_USE_SCCACHE}"]

    # not with pypy cmd += ["--limited-api=yes"]

    if is_snapshot_build():
        cmd += ["--snapshot-build"]

    qtpaths_path = get_ci_qtpaths_path(CI_ENV_INSTALL_DIR, CI_HOST_OS)
    cmd.append(qtpaths_path)

    # Due to certain older CMake versions generating very long paths
    # (at least with CMake 3.6.2) when using the export() function,
    # pass the shorter paths option on Windows so we don't hit
    # the path character length limit (260).
    if CI_HOST_OS == "Windows":
        cmd += ["--shorter-paths"]

    cmd += ["--package-timestamp=" + CI_INTEGRATION_ID]

    env = os.environ
    run_instruction(cmd, "Failed to run setup.py for build", initial_env=env)

    if phase in ["WHEEL"] or CI_RELEASE_CONF:
        cmd = [env_python, "create_wheels.py"]
        run_instruction(cmd, "Failed to create new wheels", initial_env=env)

def install_pypy():
    localfile = None
    home = expanduser("~")
    file = "https://downloads.python.org/pypy/pypy3.8-v7.3.8-osx64.tar.bz2"
    target =os.path.join(home, "work", "pypy-3.8")
    pypy = os.path.join(target, "pypy3.8-v7.3.8-osx64", "bin", "pypy")
    if sys.platform == "win32":
        file = "http://ci-files01-hki.ci.local/input/pypy/pypy3.8-v7.3.8-win64.zip"
        pypy = os.path.join(target, "pypy3.8-v7.3.8-win64", "pypy.exe")
    if CI_HOST_OS == "Linux":
        file = "https://downloads.python.org/pypy/pypy3.8-v7.3.8-linux64.tar.bz2"
        pypy = os.path.join(target, "pypy3.8-v7.3.8-linux64", "bin", "pypy")

    for i in range(1, 10):
        try:
            log.info(f"Downloading fileUrl {file}, attempt #{i}")
            localfile, info = urllib.urlretrieve(file)
            break
        except urllib.URLError:
            pass
    if not localfile:
        log.error(f"Error downloading {file} : {info}")
        raise RuntimeError(f" Error downloading {file}")

    pathlib.Path(target).mkdir(parents=True, exist_ok=True)
    if sys.platform == "win32":
        cmd = ["7z", "x", "-y", localfile, "-o"+target]
    else:
        cmd = ["tar", "xjf", localfile, "-C", target]
    run_instruction(cmd, "Failed to extract pypy")
    return  pypy


def build_with_pypy(pypy):
    run_instruction([pypy, "-m", "ensurepip"], "Failed to install pip")
    cmd = [pypy, "-m", "pip", "install", "-r", "requirements.txt"]
    run_instruction(cmd, "Failed to install requirements.txt")

    cmd = [pypy, "-m", "pip", "install", "numpy"]
    run_instruction(cmd, "Failed to install numpy")


if __name__ == "__main__":

    # Remove some environment variables that impact cmake
    arch = '32' if CI_TARGET_ARCH == 'X86' else '64'
    expand_clang_variables(arch)
    pypy = ""
    p_ver = "3"
    for env_var in ['CC', 'CXX']:
        if os.environ.get(env_var):
            del os.environ[env_var]

    wheel_package_dir = "qfpa-p3.6"

    if CI_TEST_WITH_PYPY:
        pypy = install_pypy()
        build_with_pypy(pypy)
        p_ver = "pypy"
        wheel_package_dir =  "qfp-p3.8"
    if CI_TARGET_OS in ["Windows"] and not CI_TEST_WITH_PYPY:
        if os.environ.get('HOST_OSVERSION_COIN') == 'windows_10_21h2':
            p_ver = "3.10.0"
            wheel_package_dir = "qfpa-p3.10"
        else:
            p_ver = "3.7.9"
            wheel_package_dir = "qfpa-p3.7"

    if CI_TEST_PHASE in ["ALL", "BUILD"]:
        call_setup(p_ver, "BUILD", pypy)
    # Until CI has a feature to set more dynamic signing dir, make sure it actually exist
    if os.environ.get("QTEST_ENVIRONMENT") == "ci" and sys.platform == "win32":

        signing_dir = os.path.join(os.getcwd(), "build", wheel_package_dir, "package_for_wheels")
        print("Check for signing dir " + signing_dir)
        assert(os.path.isdir(signing_dir))

    if CI_TEST_PHASE in ["ALL", "WHEEL"]:
        call_setup(p_ver,"WHEEL", pypy)