Improve libICU deployment on Linux

Previously the --standalone build process would download and extract
an archive of ICU libraries, regardless of which ICU Qt was built
against.

The build process will now detect which ICU libraries QtCore depends
on and copy the libraries over to the destintation libdir.

Something similar might be needed in the future for macOS and Windows.

Change-Id: I0db0c8c628d3c095a8a4a1e361f8fafe18da2ec3
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

2 files changed, 172 insertions, 60 deletions

diff --git a/setup.py b/setup.py
index dcf0c5bc3..e4e9a88d9 100644
--- a/setup.py
+++ b/setup.py
@@ -210,7 +210,8 @@ from utils import init_msvc_env
 from utils import regenerate_qt_resources
 from utils import filter_match
 from utils import osx_fix_rpaths_for_library
-from utils import download_and_extract_7z
+from utils import copy_icu_libs
+from utils import find_files_using_glob
 from textwrap import dedent
 
 # guess a close folder name for extensions
@@ -1160,49 +1161,27 @@ class pyside_build(_build):
     def prepare_standalone_package_linux(self, executables, vars):
         built_modules = vars['built_modules']
 
-        def _print_warn():
-            print(dedent("""\
-                  ***********************************************************
-                    If one is using Qt binaries provided by QtCompany's CI,
-                    those aren't working as expected!
-                  ***********************************************************
-                  """))
-
-        # To get working QtCompany Qt CI binaries we have to extract the ICU libs
-        # If no link provided we'll use the libs from qt-project
-        icuUrl = ""
-        if OPTION_ICULIB:
-            icuUrl = OPTION_ICULIB
-        else:
-            qt_version = self.qtinfo.version
-            url_pre = "http://master.qt.io/development_releases/prebuilt/icu/prebuilt/"
-            if qt_version.startswith("5.6"):
-                icuUrl = url_pre + "56.1/icu-linux-g++-Rhel6.6-x64.7z"
-            else:
-                icuUrl = url_pre + "56.1/icu-linux-g++-Rhel7.2-x64.7z"
-
-        if find_executable("7z"):
-            try:
-                download_and_extract_7z(icuUrl, "{qt_lib_dir}".format(**vars))
-            except RuntimeError as e:
-                # The Qt libs now requires patching to system ICU
-                # OR it is possible that Qt was built without ICU and
-                # Works as such
-                print("Failed to download and extract %s" % icuUrl)
-                print(str(e))
-                _print_warn()
-        else:
-                print("Install 7z into PATH to extract ICU libs")
-                _print_warn()
-
         # <qt>/lib/* -> <setup>/PySide2/Qt/lib
-        copydir("{qt_lib_dir}", "{pyside_package_dir}/PySide2/Qt/lib",
+        destination_lib_dir = "{pyside_package_dir}/PySide2/Qt/lib"
+        copydir("{qt_lib_dir}", destination_lib_dir,
             filter=[
                 "libQt5*.so.?",
                 "libicu*.so.??",
             ],
             recursive=False, vars=vars, force_copy_symlinks=True)
 
+        # Check if ICU libraries were copied over to the destination Qt libdir.
+        resolved_destination_lib_dir = destination_lib_dir.format(**vars)
+        maybe_icu_libs = find_files_using_glob(resolved_destination_lib_dir, "libcu*")
+
+        # If no ICU libraries are present in the Qt libdir (like when Qt is built against system
+        # ICU, or in the Coin CI where ICU libs are in a different directory) try to
+        # find out / resolve which ICU libs are used by QtCore (if used at all) using a custom
+        # written ldd, and copy the ICU libs to the Pyside Qt dir if necessary. We choose the QtCore
+        # lib to inspect, by checking which QtCore library the shiboken2 executable uses.
+        if not maybe_icu_libs:
+            copy_icu_libs(resolved_destination_lib_dir)
+
         if 'WebEngineWidgets' in built_modules:
             copydir("{qt_lib_execs_dir}", "{pyside_package_dir}/PySide2/Qt/libexec",
                 filter=None,
diff --git a/utils.py b/utils.py
index 396b79cb9..cda99bd87 100644
--- a/utils.py
+++ b/utils.py
@@ -9,6 +9,8 @@ import subprocess
 import fnmatch
 import itertools
 import popenasync
+import glob
+
 # There is no urllib.request in Python2
 try:
     import urllib.request as urllib
@@ -491,7 +493,7 @@ def regenerate_qt_resources(src, pyside_rcc_path, pyside_rcc_options):
 
 
 def back_tick(cmd, ret_err=False):
-    """ Run command `cmd`, return stdout, or stdout, stderr if `ret_err`
+    """ Run command `cmd`, return stdout, or stdout, stderr, return_code if `ret_err` is True.
 
     Roughly equivalent to ``check_output`` in Python 2.7
 
@@ -500,20 +502,20 @@ def back_tick(cmd, ret_err=False):
     cmd : str
         command to execute
     ret_err : bool, optional
-        If True, return stderr in addition to stdout.  If False, just return
+        If True, return stderr and return_code in addition to stdout.  If False, just return
         stdout
 
     Returns
     -------
     out : str or tuple
         If `ret_err` is False, return stripped string containing stdout from
-        `cmd`.  If `ret_err` is True, return tuple of (stdout, stderr) where
+        `cmd`.  If `ret_err` is True, return tuple of (stdout, stderr, return_code) where
         ``stdout`` is the stripped stdout, and ``stderr`` is the stripped
-        stderr.
+        stderr, and ``return_code`` is the process exit code.
 
     Raises
     ------
-    Raises RuntimeError if command returns non-zero exit code
+    Raises RuntimeError if command returns non-zero exit code when ret_err isn't set.
     """
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
     out, err = proc.communicate()
@@ -522,16 +524,16 @@ def back_tick(cmd, ret_err=False):
         out = out.decode()
         err = err.decode()
     retcode = proc.returncode
-    if retcode is None:
+    if retcode is None and not ret_err:
         proc.terminate()
         raise RuntimeError(cmd + ' process did not terminate')
-    if retcode != 0:
+    if retcode != 0 and not ret_err:
         raise RuntimeError(cmd + ' process returned code %d\n*** %s' %
                            (retcode, err))
     out = out.strip()
     if not ret_err:
         return out
-    return out, err.strip()
+    return out, err.strip(), retcode
 
 
 OSX_OUTNAME_RE = re.compile(r'\(compatibility version [\d.]+, current version '
@@ -662,20 +664,151 @@ def osx_add_qt_rpath(library_path, qt_lib_dir,
         back_tick('install_name_tool -add_rpath {rpath} {library_path}'.format(
                   rpath=qt_lib_dir, library_path=library_path))
 
-def download_and_extract_7z(fileurl, target):
-    """ Downloads 7z file from fileurl and extract to target  """
+def split_and_strip(input):
+    lines = [s.strip() for s in input.splitlines()]
+    return lines
 
-    print("Downloading fileUrl %s " % fileurl)
-    info = ""
-    try:
-        localfile, info = urllib.urlretrieve(fileurl)
-    except:
-        print("Error downloading %r : %r" % (fileurl, info))
-        raise RuntimeError(' Error downloading ' + fileurl)
+def ldd_get_dependencies(executable_path):
+    """ Returns a dictionary of dependencies that `executable_path` depends on.
 
-    try:
-        outputDir = "-o" + target
-        print("calling 7z x %s %s" % (localfile, outputDir))
-        subprocess.call(["7z", "x", "-y", localfile, outputDir])
-    except:
-        raise RuntimeError(' Error extracting ' + localfile)
+    The keys are library names and the values are the library paths.
+
+    """
+    output = ldd(executable_path)
+    lines = split_and_strip(output)
+    pattern = re.compile(r"\s*(.*?)\s+=>\s+(.*?)\s+\(.*\)")
+    dependencies = {}
+    for line in lines:
+        match = pattern.search(line)
+        if match:
+            dependencies[match.group(1)] = match.group(2)
+    return dependencies
+
+def ldd_get_paths_for_dependencies(dependencies_regex, executable_path = None, dependencies = None):
+    """ Returns file paths to shared library dependencies that match given `dependencies_regex`
+        against given `executable_path`.
+
+        The function retrieves the list of shared library dependencies using ld.so for the given
+        `executable_path` in order to search for libraries that match the `dependencies_regex`, and
+        then returns a list of absolute paths of the matching libraries.
+
+        If no matching library is found in the list of dependencies, an empty list is returned.
+        """
+
+    if not dependencies and not executable_path:
+        return None
+
+    if not dependencies:
+        dependencies = ldd_get_dependencies(executable_path)
+
+    pattern = re.compile(dependencies_regex)
+
+    paths = []
+    for key in dependencies:
+        match = pattern.search(key)
+        if match:
+            paths.append(dependencies[key])
+
+    return paths
+
+def ldd(executable_path):
+    """ Returns ld.so output of shared library dependencies for given `executable_path`.
+
+    This is a partial port of /usr/bin/ldd from bash to Python. The dependency list is retrieved
+    by setting the LD_TRACE_LOADED_OBJECTS=1 environment variable, and executing the given path
+    via the dynamic loader ld.so.
+
+    Only works on Linux. The port is required to make this work on systems that might not have ldd.
+    This is because ldd (on Ubuntu) is shipped in the libc-bin package that, which might have a
+    minuscule percentage of not being installed.
+
+    Parameters
+    ----------
+    executable_path : str
+        path to executable or shared library.
+
+    Returns
+    -------
+    output : str
+        the raw output retrieved from the dynamic linker.
+    """
+
+    chosen_rtld = None
+    # List of ld's considered by ldd on Ubuntu (here's hoping it's the same on all distros).
+    rtld_list = ["/lib/ld-linux.so.2", "/lib64/ld-linux-x86-64.so.2", "/libx32/ld-linux-x32.so.2"]
+
+    # Choose appropriate runtime dynamic linker.
+    for rtld in rtld_list:
+        if os.path.isfile(rtld) and os.access(rtld, os.X_OK):
+            (_, _, code) = back_tick(rtld, True)
+            # Code 127 is returned by ld.so when called without any arguments (some kind of sanity
+            # check I guess).
+            if code == 127:
+                (_, _, code) = back_tick("{} --verify {}".format(rtld, executable_path), True)
+                # Codes 0 and 2 mean given executable_path can be understood by ld.so.
+                if code in [0, 2]:
+                    chosen_rtld = rtld
+                    break
+
+    if not chosen_rtld:
+        raise RuntimeError('Could not find appropriate ld.so to query for dependencies.')
+
+    # Query for shared library dependencies.
+    rtld_env = "LD_TRACE_LOADED_OBJECTS=1"
+    rtld_cmd = "{} {} {}".format(rtld_env, chosen_rtld, executable_path)
+    (out, _, return_code) = back_tick(rtld_cmd, True)
+    if return_code == 0:
+        return out
+    else:
+        raise RuntimeError('ld.so failed to query for dependent shared libraries '
+                           'of {} '.format(executable_path))
+
+def find_files_using_glob(path, pattern):
+    """ Returns list of files that matched glob `pattern` in `path`. """
+    final_pattern = os.path.join(path, pattern)
+    maybe_files = glob.glob(final_pattern)
+    return maybe_files
+
+def find_qt_core_library_glob(lib_dir):
+    """ Returns path to the QtCore library found in `lib_dir`. """
+    maybe_file = find_files_using_glob(lib_dir, "libQt5Core.so.?")
+    if len(maybe_file) == 1:
+        return maybe_file[0]
+    return None
+
+# @TODO: Possibly fix ICU library copying on macOS and Windows. This would require
+# to implement the equivalent of the custom written ldd for the specified platforms.
+# This has less priority because ICU libs are not used in the default Qt configuration build.
+
+def copy_icu_libs(destination_lib_dir):
+    """ Copy ICU libraries that QtCore depends on, to given `destination_lib_dir`. """
+    qt_core_library_path = find_qt_core_library_glob(destination_lib_dir)
+
+    if not qt_core_library_path or not os.path.exists(qt_core_library_path):
+        raise RuntimeError('QtCore library does not exist at path: {}. '
+                           'Failed to copy ICU libraries.'.format(qt_core_library_path))
+
+    dependencies = ldd_get_dependencies(qt_core_library_path)
+
+    icu_regex = r"^libicu.+"
+    icu_compiled_pattern = re.compile(icu_regex)
+    icu_required = False
+    for dependency in dependencies:
+        match = icu_compiled_pattern.search(dependency)
+        if match:
+            icu_required = True
+            break
+
+    if icu_required:
+        paths = ldd_get_paths_for_dependencies(icu_regex, dependencies=dependencies)
+        if not paths:
+            raise RuntimeError('Failed to find the necessary ICU libraries required by QtCore.')
+        log.info('Copying the detected ICU libraries required by QtCore.')
+
+        if not os.path.exists(destination_lib_dir):
+            os.makedirs(destination_lib_dir)
+
+        for path in paths:
+            basename = os.path.basename(path)
+            destination = os.path.join(destination_lib_dir, basename)
+            copyfile(path, destination, force_copy_symlink=True)