From 95a5bb9dd3b5d3fa86f2ed0868e2b821256a6028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristi=C3=A1n=20Maureira-Fredes?= Date: Thu, 2 Sep 2021 13:44:17 +0200 Subject: Move from distutils to setuptools This is motivated by the deprecation of distutils, and removal in future versions https://github.com/pypa/packaging-problems/issues/127 Pick-to: 6.2 Change-Id: I16448b69f98df6dc1d9a904b69eb69ed5f1093f5 Reviewed-by: Christian Tismer --- build_scripts/config.py | 2 +- build_scripts/main.py | 80 ++++++++++++---------- build_scripts/options.py | 16 +++-- build_scripts/setup_runner.py | 2 +- build_scripts/utils.py | 59 ++++++++-------- build_scripts/wheel_override.py | 2 +- build_scripts/wheel_utils.py | 9 ++- coin/instructions/execute_build_instructions.yaml | 10 +++ examples/utils/pyside_config.py | 4 +- requirements.txt | 2 +- sources/pyside6/cmake/PySideSetup.cmake | 16 ++++- sources/shiboken6/cmake/ShibokenHelpers.cmake | 14 +++- .../shiboken6/tests/samplebinding/numpy_test.py | 2 +- testing/wheel_tester.py | 2 +- 14 files changed, 132 insertions(+), 88 deletions(-) diff --git a/build_scripts/config.py b/build_scripts/config.py index 320b2b80a..7a42f7cc7 100644 --- a/build_scripts/config.py +++ b/build_scripts/config.py @@ -38,7 +38,7 @@ ############################################################################# import os -import distutils.log as log +from setuptools._distutils import log from .versions import PYSIDE, PYSIDE_MODULE, SHIBOKEN diff --git a/build_scripts/main.py b/build_scripts/main.py index 92a0ac1f4..667ed8c68 100644 --- a/build_scripts/main.py +++ b/build_scripts/main.py @@ -37,7 +37,7 @@ ## ############################################################################# -from distutils.version import LooseVersion +from packaging.version import parse as parse_version import os import platform @@ -53,6 +53,33 @@ from .wheel_utils import (get_package_version, get_qt_version, get_package_timestamp, macos_plat_name, macos_pyside_min_deployment_target) +import setuptools # Import setuptools before distutils +from setuptools import Extension +from setuptools.command.install import install as _install +from setuptools.command.install_lib import install_lib as _install_lib +from setuptools.command.bdist_egg import bdist_egg as _bdist_egg +from setuptools.command.develop import develop as _develop +from setuptools.command.build_py import build_py as _build_py + +from sysconfig import get_config_var +# Use the distutils implementation within setuptools +from setuptools._distutils.errors import DistutilsError +from setuptools._distutils import log +from setuptools._distutils import sysconfig as sconfig +from setuptools._distutils.command.build import build as _build + +from shutil import which +from setuptools.command.build_ext import build_ext as _build_ext +from setuptools import Command + +from .qtinfo import QtInfo +from .utils import (rmtree, detect_clang, copyfile, copydir, run_process_output, run_process, + update_env_path, init_msvc_env, filter_match, macos_fix_rpaths_for_library, + linux_fix_rpaths_for_library) +from .platforms.unix import prepare_packages_posix +from .platforms.windows_desktop import prepare_packages_win32 +from .wheel_override import wheel_module_exists, get_bdist_wheel_override + setup_script_dir = os.getcwd() build_scripts_dir = os.path.join(setup_script_dir, 'build_scripts') @@ -84,15 +111,15 @@ def _get_make(platform_arch, build_type): if makespec == "make": return ("make", "Unix Makefiles") if makespec == "msvc": - nmake_path = find_executable("nmake") + nmake_path = which("nmake") if nmake_path is None or not os.path.exists(nmake_path): log.info("nmake not found. Trying to initialize the MSVC env...") init_msvc_env(platform_arch, build_type) - nmake_path = find_executable("nmake") + nmake_path = which("nmake") if not nmake_path or not os.path.exists(nmake_path): raise DistutilsSetupError('"nmake" could not be found.') if not OPTION["NO_JOM"]: - jom_path = find_executable("jom") + jom_path = which("jom") if jom_path: log.info(f"jom was found in {jom_path}") return (jom_path, "NMake Makefiles JOM") @@ -112,7 +139,7 @@ def get_make(platform_arch, build_type): """Retrieve the make command and CMake generator name""" (make_path, make_generator) = _get_make(platform_arch, build_type) if not os.path.isabs(make_path): - found_path = find_executable(make_path) + found_path = which(make_path) if not found_path or not os.path.exists(found_path): m = f"You need the program '{make_path}' on your system path to compile {PYSIDE_MODULE}." raise DistutilsSetupError(m) @@ -208,39 +235,11 @@ def get_py_library(build_type, py_version, py_prefix, py_libdir, py_include_dir) return py_library -import setuptools # Import setuptools before distutils -from setuptools import Extension -from setuptools.command.install import install as _install -from setuptools.command.install_lib import install_lib as _install_lib -from setuptools.command.bdist_egg import bdist_egg as _bdist_egg -from setuptools.command.develop import develop as _develop -from setuptools.command.build_py import build_py as _build_py - -import distutils.log as log -from distutils.errors import DistutilsSetupError -from distutils.sysconfig import get_config_var -from distutils.sysconfig import get_python_lib -from distutils.spawn import find_executable -from distutils.command.build import build as _build -from distutils.command.build_ext import build_ext as _build_ext -from distutils.cmd import Command - -from .qtinfo import QtInfo -from .utils import rmtree, detect_clang, copyfile, copydir, run_process_output, run_process -from .utils import update_env_path, init_msvc_env, filter_match -from .utils import macos_fix_rpaths_for_library -from .utils import linux_fix_rpaths_for_library -from .platforms.unix import prepare_packages_posix -from .platforms.windows_desktop import prepare_packages_win32 -from .wheel_override import wheel_module_exists, get_bdist_wheel_override - - def check_allowed_python_version(): """ Make sure that setup.py is run with an allowed python version. """ - import re pattern = r'Programming Language :: Python :: (\d+)\.(\d+)' supported = [] @@ -463,7 +462,12 @@ class PysideBuild(_build, DistUtilsCommandMixin): py_version = f"{sys.version_info[0]}.{sys.version_info[1]}" py_include_dir = get_config_var("INCLUDEPY") py_libdir = get_config_var("LIBDIR") - py_prefix = get_config_var("prefix") + # distutils.sysconfig.get_config_var('prefix') returned the + # virtual environment base directory, but + # sysconfig.get_config_var returns the system's prefix. + # We use 'base' instead (although, platbase points to the + # same location) + py_prefix = get_config_var("base") if not py_prefix or not os.path.exists(py_prefix): py_prefix = sys.prefix self.py_prefix = py_prefix @@ -485,7 +489,7 @@ class PysideBuild(_build, DistUtilsCommandMixin): # Add Clang to path for Windows. # Revisit once Clang is bundled with Qt. if (sys.platform == "win32" - and LooseVersion(self.qtinfo.version) >= LooseVersion("5.7.0")): + and parse_version(self.qtinfo.version) >= parse_version("5.7.0")): clang_dir = detect_clang() if clang_dir[0]: clangBinDir = os.path.join(clang_dir[0], 'bin') @@ -528,7 +532,7 @@ class PysideBuild(_build, DistUtilsCommandMixin): py_libdir, py_include_dir) self.py_version = py_version self.build_type = build_type - self.site_packages_dir = get_python_lib(1, 0, prefix=install_dir) + self.site_packages_dir = sconfig.get_python_lib(1, 0, prefix=install_dir) self.build_tests = OPTION["BUILDTESTS"] # Save the shiboken build dir path for clang deployment @@ -584,7 +588,7 @@ class PysideBuild(_build, DistUtilsCommandMixin): if config.is_internal_shiboken_generator_build_and_part_of_top_level_all(): return - setuptools_install_prefix = get_python_lib(1) + setuptools_install_prefix = sconfig.get_python_lib(1) if OPTION["FINAL_INSTALL_PREFIX"]: setuptools_install_prefix = OPTION["FINAL_INSTALL_PREFIX"] log.info("=" * 30) @@ -640,7 +644,7 @@ class PysideBuild(_build, DistUtilsCommandMixin): def build_patchelf(self): if not sys.platform.startswith('linux'): return - self._patchelf_path = find_executable('patchelf') + self._patchelf_path = which('patchelf') if self._patchelf_path: if not os.path.isabs(self._patchelf_path): self._patchelf_path = os.path.join(os.getcwd(), self._patchelf_path) diff --git a/build_scripts/options.py b/build_scripts/options.py index 068bd62ab..ff1260586 100644 --- a/build_scripts/options.py +++ b/build_scripts/options.py @@ -37,8 +37,14 @@ ## ############################################################################# -import distutils.log as log -from distutils.spawn import find_executable +try: + from setuptools._distutils import log +except ModuleNotFoundError: + # This is motivated by our CI using an old version of setuptools + # so then the coin_build_instructions.py script is executed, and + # import from this file, it was failing. + from distutils import log +from shutil import which import sys import os import warnings @@ -334,7 +340,7 @@ class DistUtilsCommandMixin(object): def _determine_defaults_and_check(self): if not self.cmake: - self.cmake = find_executable("cmake") + self.cmake = which("cmake") if not self.cmake: log.error("cmake could not be found.") return False @@ -343,14 +349,14 @@ class DistUtilsCommandMixin(object): return False if not self.qtpaths: - self.qtpaths = find_executable("qtpaths") + self.qtpaths = which("qtpaths") if not self.qtpaths: self.qtpaths = find_executable("qtpaths6") if self.qmake: self.has_qmake_option = True else: - self.qmake = find_executable("qmake") + self.qmake = which("qmake") if not self.qmake: self.qmake = find_executable("qmake6") if not self.qmake: diff --git a/build_scripts/setup_runner.py b/build_scripts/setup_runner.py index 846cc67f0..ddf98fef0 100644 --- a/build_scripts/setup_runner.py +++ b/build_scripts/setup_runner.py @@ -42,7 +42,7 @@ import os import textwrap from setuptools import setup # Import setuptools before distutils -import distutils.log as log +from setuptools._distutils import log from build_scripts.config import config from build_scripts.main import get_package_version, get_setuptools_extension_modules diff --git a/build_scripts/utils.py b/build_scripts/utils.py index 45d7481cf..4b91bd493 100644 --- a/build_scripts/utils.py +++ b/build_scripts/utils.py @@ -51,8 +51,17 @@ import glob import urllib.request as urllib -import distutils.log as log -from distutils.errors import DistutilsSetupError + +try: + # Using the distutils implementation within setuptools + from setuptools._distutils import log + from setuptools._distutils.errors import DistutilsError +except ModuleNotFoundError: + # This is motivated by our CI using an old version of setuptools + # so then the coin_build_instructions.py script is executed, and + # import from this file, it was failing. + from distutils import log + from distutils import errors try: WindowsError @@ -88,10 +97,7 @@ def get_numpy_location(): def winsdk_setenv(platform_arch, build_type): - from distutils.msvc9compiler import VERSION as MSVC_VERSION - from distutils.msvc9compiler import Reg - from distutils.msvc9compiler import HKEYS - from distutils.msvc9compiler import WINSDK_BASE + from setuptools._distutils import msvc9compiler as msvc9 sdk_version_map = { "v6.0a": 9.0, @@ -101,27 +107,27 @@ def winsdk_setenv(platform_arch, build_type): "v7.1": 10.0 } - log.info(f"Searching Windows SDK with MSVC compiler version {MSVC_VERSION}") + log.info(f"Searching Windows SDK with MSVC compiler version {msvc9.VERSION}") setenv_paths = [] - for base in HKEYS: - sdk_versions = Reg.read_keys(base, WINSDK_BASE) + for base in msvc9.HKEYS: + sdk_versions = msvc9.Reg.read_keys(base, msvc9.WINSDK_BASE) if sdk_versions: for sdk_version in sdk_versions: - installationfolder = Reg.get_value(f"{WINSDK_BASE}\\{sdk_version}", + installationfolder = msvc9.Reg.get_value(f"{msvc9.WINSDK_BASE}\\{sdk_version}", "installationfolder") - # productversion = Reg.get_value("{}\\{}".format(WINSDK_BASE, sdk_version), + # productversion = msvc9.Reg.get_value("{}\\{}".format(msvc9.WINSDK_BASE, sdk_version), # "productversion") setenv_path = os.path.join(installationfolder, os.path.join('bin', 'SetEnv.cmd')) if not os.path.exists(setenv_path): continue if sdk_version not in sdk_version_map: continue - if sdk_version_map[sdk_version] != MSVC_VERSION: + if sdk_version_map[sdk_version] != msvc9.VERSION: continue setenv_paths.append(setenv_path) if len(setenv_paths) == 0: raise DistutilsSetupError("Failed to find the Windows SDK with MSVC compiler " - f"version {MSVC_VERSION}") + f"version {msvc9.VERSION}") for setenv_path in setenv_paths: log.info(f"Found {setenv_path}") @@ -148,26 +154,25 @@ def winsdk_setenv(platform_arch, build_type): def find_vcdir(version): """ This is the customized version of - distutils.msvc9compiler.find_vcvarsall method + setuptools._distutils.msvc9compiler.find_vcvarsall method """ - from distutils.msvc9compiler import VS_BASE - from distutils.msvc9compiler import Reg - vsbase = VS_BASE % version + from setuptools._distutils import msvc9compiler as msvc9 + vsbase = msvc9.VS_BASE % version try: - productdir = Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") + productdir = msvc9.Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") except KeyError: productdir = None # trying Express edition if productdir is None: try: - from distutils.msvc9compiler import VSEXPRESS_BASE - except ImportError: + hasattr(msvc9, VSEXPRESS_BASE) + except AttributeError: pass else: vsbase = VSEXPRESS_BASE % version try: - productdir = Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") + productdir = msvc9.Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") except KeyError: productdir = None log.debug("Unable to find productdir in registry") @@ -191,17 +196,17 @@ def find_vcdir(version): def init_msvc_env(platform_arch, build_type): - from distutils.msvc9compiler import VERSION as MSVC_VERSION + from setuptools._distutils import msvc9compiler as msvc9 - log.info(f"Searching MSVC compiler version {MSVC_VERSION}") - vcdir_path = find_vcdir(MSVC_VERSION) + log.info(f"Searching MSVC compiler version {msvc9.VERSION}") + vcdir_path = find_vcdir(msvc9.VERSION) if not vcdir_path: - raise DistutilsSetupError(f"Failed to find the MSVC compiler version {MSVC_VERSION} on " + raise DistutilsSetupError(f"Failed to find the MSVC compiler version {msvc9.VERSION} on " "your system.") else: log.info(f"Found {vcdir_path}") - log.info(f"Searching MSVC compiler {MSVC_VERSION} environment init script") + log.info(f"Searching MSVC compiler {msvc9.VERSION} environment init script") if platform_arch.startswith("32"): vcvars_path = os.path.join(vcdir_path, "bin", "vcvars32.bat") else: @@ -221,7 +226,7 @@ def init_msvc_env(platform_arch, build_type): log.info(f"Found {vcvars_path}") # Get MSVC env - log.info(f"Using MSVC {MSVC_VERSION} in {vcvars_path}") + log.info(f"Using MSVC {msvc9.VERSION} in {vcvars_path}") msvc_arch = "x86" if platform_arch.startswith("32") else "amd64" log.info(f"Getting MSVC env for {msvc_arch} architecture") vcvars_cmd = [vcvars_path, msvc_arch] diff --git a/build_scripts/wheel_override.py b/build_scripts/wheel_override.py index 0811a580b..4706863d5 100644 --- a/build_scripts/wheel_override.py +++ b/build_scripts/wheel_override.py @@ -43,7 +43,7 @@ wheel_module_exists = False import os import sys from .options import DistUtilsCommandMixin, OPTION -from distutils import log as logger +from setuptools._distutils import log as logger from email.generator import Generator from .wheel_utils import get_package_version, get_qt_version, macos_plat_name diff --git a/build_scripts/wheel_utils.py b/build_scripts/wheel_utils.py index cea45b107..4cea2a83f 100644 --- a/build_scripts/wheel_utils.py +++ b/build_scripts/wheel_utils.py @@ -40,10 +40,9 @@ import os import time -from distutils.errors import DistutilsSetupError -from distutils.sysconfig import get_config_var -from distutils.util import get_platform -from distutils.version import LooseVersion +from setuptools._distutils.errors import DistutilsError +from sysconfig import get_platform, get_config_var +from packaging.version import parse as parse_version from .options import OPTION from .qtinfo import QtInfo @@ -68,7 +67,7 @@ def get_qt_version(): raise DistutilsSetupError("Failed to query the Qt version with " f"qmake {qtinfo.qmake_command}") - if LooseVersion(qtinfo.version) < LooseVersion("5.7"): + if parse_version(qtinfo.version) < parse_version("5.7"): raise DistutilsSetupError(f"Incompatible Qt version detected: {qt_version}. " "A Qt version >= 5.7 is required.") diff --git a/coin/instructions/execute_build_instructions.yaml b/coin/instructions/execute_build_instructions.yaml index 8b1789728..e1a082bb6 100644 --- a/coin/instructions/execute_build_instructions.yaml +++ b/coin/instructions/execute_build_instructions.yaml @@ -1,5 +1,15 @@ type: Group instructions: + - type: ExecuteCommand + command: "python3 -m pip install -U setuptools" + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 1200 + enable_if: + condition: property + property: host.os + equals_value: MacOS + userMessageOnFailure: > + Failed to install setuptools on osx - type: ExecuteCommand command: "python3 -u coin_build_instructions.py --os={{.Env.CI_OS}} {{.Env.CI_PACKAGING_FEATURE}} {{.Env.CI_USE_SCCACHE}} --instdir=/Users/qt/work/install --targetOs={{.Env.CI_OS}} --hostArch=X86_64 --targetArch={{.Env.CI_TARGET_ARCHITECTURE}} --phase=ALL" maxTimeInSeconds: 14400 diff --git a/examples/utils/pyside_config.py b/examples/utils/pyside_config.py index 02d4a5da2..6a3338b17 100644 --- a/examples/utils/pyside_config.py +++ b/examples/utils/pyside_config.py @@ -38,7 +38,7 @@ ## ############################################################################# -from distutils import sysconfig +import sysconfig from enum import Enum import glob import os @@ -237,7 +237,7 @@ def python_version(): def get_python_include_path(): - return sysconfig.get_python_inc() + return sysconfig.get_path('include') def python_link_flags_qmake(): diff --git a/requirements.txt b/requirements.txt index 7141c9df7..a2b45fbc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -setuptools +setuptools>=48.0 sphinx sphinx-panels six diff --git a/sources/pyside6/cmake/PySideSetup.cmake b/sources/pyside6/cmake/PySideSetup.cmake index 4f7c71626..1d5e13cad 100644 --- a/sources/pyside6/cmake/PySideSetup.cmake +++ b/sources/pyside6/cmake/PySideSetup.cmake @@ -207,9 +207,19 @@ endif() if(NOT PYTHON_SITE_PACKAGES) execute_process( COMMAND ${SHIBOKEN_PYTHON_INTERPRETER} -c "if True: - from distutils import sysconfig - from os import sep - print(sysconfig.get_python_lib(1, 0, prefix='${CMAKE_INSTALL_PREFIX}').replace(sep, '/')) + import sysconfig + from os.path import sep + + # /home/qt/dev/env/lib/python3.9/site-packages + lib_path = sysconfig.get_path('purelib') + + # /home/qt/dev/env + data_path = sysconfig.get_path('data') + + # /lib/python3.9/site-packages + rel_path = lib_path.replace(data_path, '') + + print(f'${CMAKE_INSTALL_PREFIX}{rel_path}'.replace(sep, '/')) " OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/sources/shiboken6/cmake/ShibokenHelpers.cmake b/sources/shiboken6/cmake/ShibokenHelpers.cmake index c17401ccc..87a676219 100644 --- a/sources/shiboken6/cmake/ShibokenHelpers.cmake +++ b/sources/shiboken6/cmake/ShibokenHelpers.cmake @@ -78,9 +78,19 @@ endmacro() macro(set_python_site_packages) execute_process( COMMAND ${PYTHON_EXECUTABLE} -c "if True: - from distutils import sysconfig + import sysconfig from os.path import sep - print(sysconfig.get_python_lib(1, 0, prefix='${CMAKE_INSTALL_PREFIX}').replace(sep, '/')) + + # /home/qt/dev/env/lib/python3.9/site-packages + lib_path = sysconfig.get_path('purelib') + + # /home/qt/dev/env + data_path = sysconfig.get_path('data') + + # /lib/python3.9/site-packages + rel_path = lib_path.replace(data_path, '') + + print(f'${CMAKE_INSTALL_PREFIX}{rel_path}'.replace(sep, '/')) " OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/sources/shiboken6/tests/samplebinding/numpy_test.py b/sources/shiboken6/tests/samplebinding/numpy_test.py index 61f8c7103..be13fdfda 100644 --- a/sources/shiboken6/tests/samplebinding/numpy_test.py +++ b/sources/shiboken6/tests/samplebinding/numpy_test.py @@ -29,7 +29,7 @@ import sys try: - from distutils import sysconfig + import sysconfig if bool(sysconfig.get_config_var('Py_DEBUG')): sys.exit(0) import numpy diff --git a/testing/wheel_tester.py b/testing/wheel_tester.py index 567d2380e..5dfb73041 100644 --- a/testing/wheel_tester.py +++ b/testing/wheel_tester.py @@ -71,7 +71,7 @@ from build_scripts.utils import find_files_using_glob from build_scripts.utils import find_glob_in_path from build_scripts.utils import run_process, run_process_output from build_scripts.utils import rmtree -import distutils.log as log +from setuptools._distutils import log import platform log.set_verbosity(1) -- cgit v1.2.3