Host and use our own copy of "virtualenv --relocatable"

With virtualenv 20, the --relocatable flag got removed
and causes build failures in QtIvi. As we still need
this functionality for making our virtualenv work
after 'make install', we copied the legacy code and
converted it into a standalone python script which
does the job for now.

With virtualenv 20 the complete deploy mechanism changed
and is now based on top on venv. To make the new environment
relocatable we need to copy additional files as well as
read the pyvenv.cfg instead of the old-prefix.txt

Change-Id: Ib77bbd2a1c959693bf1ae6ca478086ce2c848036
Fixes: AUTOSUITE-1482
Reviewed-by: Robert Griebl <robert.griebl@qt.io>
(cherry picked from commit f2be418701f451a4d68a4949d996b873b1836305)
(cherry picked from commit 721f1e60cefdaa1a17eb945a6b806fdfb9797546)
Reviewed-by: Jukka Jokiniva <jukka.jokiniva@qt.io>

8 files changed, 406 insertions, 10 deletions

diff --git a/mkspecs/features/ivigenerator.prf b/mkspecs/features/ivigenerator.prf
index 88f5e9a..7f3b9ea 100644
--- a/mkspecs/features/ivigenerator.prf
+++ b/mkspecs/features/ivigenerator.prf
@@ -62,9 +62,12 @@ qtConfig(system-qface) {
 } else {
     equals(QMAKE_HOST.os, Windows) {
         PYTHON = $$VIRTUALENV_PATH/Scripts/python.exe
+        # No space between the path and && otherwise python3 adds the space to the path
+        ENV += set PYTHONHOME=$$system_path($$VIRTUALENV_PATH)&&
     } else {
         PYTHON = $$VIRTUALENV_PATH/bin/python
         ENV += LD_LIBRARY_PATH="$$shell_path($$VIRTUALENV_PATH/bin)"
+        ENV += PYTHONHOME=$$system_path($$VIRTUALENV_PATH)
     }
 }
 IVI_GENERATOR = $$shell_quote($$PYTHON) $$IVI_GENERATOR_PATH/generate.py
diff --git a/src/3rdparty/virtualenv/VIRTUALENV_LICENSE b/src/3rdparty/virtualenv/VIRTUALENV_LICENSE
new file mode 100644
index 0000000..ab14500
--- /dev/null
+++ b/src/3rdparty/virtualenv/VIRTUALENV_LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) 2007 Ian Bicking and Contributors
+Copyright (c) 2009 Ian Bicking, The Open Planning Project
+Copyright (c) 2011-2016 The virtualenv developers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/3rdparty/virtualenv/qt_attribution.json b/src/3rdparty/virtualenv/qt_attribution.json
new file mode 100644
index 0000000..2c84b64
--- /dev/null
+++ b/src/3rdparty/virtualenv/qt_attribution.json
@@ -0,0 +1,17 @@
+{
+    "Id": "relocate-virtualenv",
+    "Name": "relocate-virtualenv",
+    "QDocModule": "qtivi",
+    "QtUsage": "Used to make the python3 virtualen relocatable, needed for the ivigenerator",
+    "Path": "relocate-virtualenv.py",
+
+    "Description": "A tool for creating isolated virtual python environments.",
+    "Homepage": "https://virtualenv.pypa.io/en/stable/",
+    "Version": "16.7.9",
+
+    "License": "MIT License",
+    "LicenseId": "MIT",
+    "LicenseFile": "VIRTUALENV_LICENSE",
+    "Copyright": "(c) 2016 by the virtualenv Team, see AUTHORS for more details."
+}
+
diff --git a/src/3rdparty/virtualenv/relocate_virtualenv.py b/src/3rdparty/virtualenv/relocate_virtualenv.py
new file mode 100644
index 0000000..3cfedeb
--- /dev/null
+++ b/src/3rdparty/virtualenv/relocate_virtualenv.py
@@ -0,0 +1,301 @@
+#!/usr/bin/env python
+"""Makes a python installation relocatable"""
+
+# Copied from https://github.com/pypa/virtualenv/blob/legacy/virtualenv.py
+# Newer virtualenv versions don't provide this functionality anymore
+
+import os
+import sys
+import logging
+from os.path import join
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger()
+
+VERSION = "{}.{}".format(*sys.version_info)
+PY_VERSION = "python{}.{}".format(*sys.version_info)
+
+IS_PYPY = hasattr(sys, "pypy_version_info")
+IS_WIN = sys.platform == "win32"
+IS_CYGWIN = sys.platform == "cygwin"
+IS_DARWIN = sys.platform == "darwin"
+ABI_FLAGS = getattr(sys, "abiflags", "")
+
+# Relocating the environment:
+def make_environment_relocatable(home_dir):
+    """
+    Makes the already-existing environment use relative paths, and takes out
+    the #!-based environment selection in scripts.
+    """
+    home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)
+    activate_this = os.path.join(bin_dir, "activate_this.py")
+    if not os.path.exists(activate_this):
+        logger.fatal(
+            "The environment doesn't have a file %s -- please re-run virtualenv " "on this environment to update it",
+            activate_this,
+        )
+    fixup_scripts(home_dir, bin_dir)
+    fixup_pth_and_egg_link(home_dir)
+    # FIXME: need to fix up distutils.cfg
+
+
+OK_ABS_SCRIPTS = [
+    "python",
+    PY_VERSION,
+    "activate",
+    "activate.bat",
+    "activate_this.py",
+    "activate.fish",
+    "activate.csh",
+    "activate.xsh",
+]
+
+
+def mkdir(at_path):
+    if not os.path.exists(at_path):
+        logger.info("Creating %s", at_path)
+        os.makedirs(at_path)
+    else:
+        logger.info("Directory %s already exists", at_path)
+
+def fixup_scripts(_, bin_dir):
+    if IS_WIN:
+        new_shebang_args = ("{} /c".format(os.path.normcase(os.environ.get("COMSPEC", "cmd.exe"))), "", ".exe")
+    else:
+        new_shebang_args = ("/usr/bin/env", VERSION, "")
+
+    # This is what we expect at the top of scripts:
+    shebang = "#!{}".format(
+        os.path.normcase(os.path.join(os.path.abspath(bin_dir), "python{}".format(new_shebang_args[2])))
+    )
+    # This is what we'll put:
+    new_shebang = "#!{} python{}{}".format(*new_shebang_args)
+
+    for filename in os.listdir(bin_dir):
+        filename = os.path.join(bin_dir, filename)
+        if not os.path.isfile(filename):
+            # ignore child directories, e.g. .svn ones.
+            continue
+        with open(filename, "rb") as f:
+            try:
+                lines = f.read().decode("utf-8").splitlines()
+            except UnicodeDecodeError:
+                # This is probably a binary program instead
+                # of a script, so just ignore it.
+                continue
+        if not lines:
+            logger.warning("Script %s is an empty file", filename)
+            continue
+
+        old_shebang = lines[0].strip()
+        old_shebang = old_shebang[0:2] + os.path.normcase(old_shebang[2:])
+
+        if not old_shebang.startswith(shebang):
+            if os.path.basename(filename) in OK_ABS_SCRIPTS:
+                logger.debug("Cannot make script %s relative", filename)
+            elif lines[0].strip() == new_shebang:
+                logger.info("Script %s has already been made relative", filename)
+            else:
+                logger.warning(
+                    "Script %s cannot be made relative (it's not a normal script that starts with %s)",
+                    filename,
+                    shebang,
+                )
+            continue
+        logger.info("Making script %s relative", filename)
+        #logger.notify("Making script %s relative", filename)
+        script = relative_script([new_shebang] + lines[1:])
+        with open(filename, "wb") as f:
+            f.write("\n".join(script).encode("utf-8"))
+
+
+def relative_script(lines):
+    """Return a script that'll work in a relocatable environment."""
+    activate = (
+        "import os; "
+        "activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); "
+        "exec(compile(open(activate_this).read(), activate_this, 'exec'), { '__file__': activate_this}); "
+        "del os, activate_this"
+    )
+    # Find the last future statement in the script. If we insert the activation
+    # line before a future statement, Python will raise a SyntaxError.
+    activate_at = None
+    for idx, line in reversed(list(enumerate(lines))):
+        if line.split()[:3] == ["from", "__future__", "import"]:
+            activate_at = idx + 1
+            break
+    if activate_at is None:
+        # Activate after the shebang.
+        activate_at = 1
+    return lines[:activate_at] + ["", activate, ""] + lines[activate_at:]
+
+
+def fixup_pth_and_egg_link(home_dir, sys_path=None):
+    """Makes .pth and .egg-link files use relative paths"""
+    home_dir = os.path.normcase(os.path.abspath(home_dir))
+    if sys_path is None:
+        sys_path = sys.path
+    for a_path in sys_path:
+        if not a_path:
+            a_path = "."
+        if not os.path.isdir(a_path):
+            continue
+        a_path = os.path.normcase(os.path.abspath(a_path))
+        if not a_path.startswith(home_dir):
+            logger.debug("Skipping system (non-environment) directory %s", a_path)
+            continue
+        for filename in os.listdir(a_path):
+            filename = os.path.join(a_path, filename)
+            if filename.endswith(".pth"):
+                if not os.access(filename, os.W_OK):
+                    logger.warning("Cannot write .pth file %s, skipping", filename)
+                else:
+                    fixup_pth_file(filename)
+            if filename.endswith(".egg-link"):
+                if not os.access(filename, os.W_OK):
+                    logger.warning("Cannot write .egg-link file %s, skipping", filename)
+                else:
+                    fixup_egg_link(filename)
+
+
+def fixup_pth_file(filename):
+    lines = []
+    with open(filename) as f:
+        prev_lines = f.readlines()
+    for line in prev_lines:
+        line = line.strip()
+        if not line or line.startswith("#") or line.startswith("import ") or os.path.abspath(line) != line:
+            lines.append(line)
+        else:
+            new_value = make_relative_path(filename, line)
+            if line != new_value:
+                logger.debug("Rewriting path {} as {} (in {})".format(line, new_value, filename))
+            lines.append(new_value)
+    if lines == prev_lines:
+        logger.info("No changes to .pth file %s", filename)
+        return
+    logger.notify("Making paths in .pth file %s relative", filename)
+    with open(filename, "w") as f:
+        f.write("\n".join(lines) + "\n")
+
+
+def fixup_egg_link(filename):
+    with open(filename) as f:
+        link = f.readline().strip()
+    if os.path.abspath(link) != link:
+        logger.debug("Link in %s already relative", filename)
+        return
+    new_link = make_relative_path(filename, link)
+    logger.notify("Rewriting link {} in {} as {}".format(link, filename, new_link))
+    with open(filename, "w") as f:
+        f.write(new_link)
+
+
+def make_relative_path(source, dest, dest_is_directory=True):
+    """
+    Make a filename relative, where the filename is dest, and it is
+    being referred to from the filename source.
+        >>> make_relative_path('/usr/share/something/a-file.pth',
+        ...                    '/usr/share/another-place/src/Directory')
+        '../another-place/src/Directory'
+        >>> make_relative_path('/usr/share/something/a-file.pth',
+        ...                    '/home/user/src/Directory')
+        '../../../home/user/src/Directory'
+        >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/')
+        './'
+    """
+    source = os.path.dirname(source)
+    if not dest_is_directory:
+        dest_filename = os.path.basename(dest)
+        dest = os.path.dirname(dest)
+    else:
+        dest_filename = None
+    dest = os.path.normpath(os.path.abspath(dest))
+    source = os.path.normpath(os.path.abspath(source))
+    dest_parts = dest.strip(os.path.sep).split(os.path.sep)
+    source_parts = source.strip(os.path.sep).split(os.path.sep)
+    while dest_parts and source_parts and dest_parts[0] == source_parts[0]:
+        dest_parts.pop(0)
+        source_parts.pop(0)
+    full_parts = [".."] * len(source_parts) + dest_parts
+    if not dest_is_directory and dest_filename is not None:
+        full_parts.append(dest_filename)
+    if not full_parts:
+        # Special case for the current directory (otherwise it'd be '')
+        return "./"
+    return os.path.sep.join(full_parts)
+
+def path_locations(home_dir, dry_run=False):
+    """Return the path locations for the environment (where libraries are,
+    where scripts go, etc)"""
+    home_dir = os.path.abspath(home_dir)
+    lib_dir, inc_dir, bin_dir = None, None, None
+    # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its
+    # prefix arg is broken: http://bugs.python.org/issue3386
+    if IS_WIN:
+        # Windows has lots of problems with executables with spaces in
+        # the name; this function will remove them (using the ~1
+        # format):
+        if not dry_run:
+            mkdir(home_dir)
+        if " " in home_dir:
+            import ctypes
+
+            get_short_path_name = ctypes.windll.kernel32.GetShortPathNameW
+            size = max(len(home_dir) + 1, 256)
+            buf = ctypes.create_unicode_buffer(size)
+            try:
+                # noinspection PyUnresolvedReferences
+                u = unicode
+            except NameError:
+                u = str
+            ret = get_short_path_name(u(home_dir), buf, size)
+            if not ret:
+                print('Error: the path "{}" has a space in it'.format(home_dir))
+                print("We could not determine the short pathname for it.")
+                print("Exiting.")
+                sys.exit(3)
+            home_dir = str(buf.value)
+        lib_dir = join(home_dir, "Lib")
+        inc_dir = join(home_dir, "Include")
+        bin_dir = join(home_dir, "Scripts")
+    if IS_PYPY:
+        lib_dir = home_dir
+        inc_dir = join(home_dir, "include")
+        bin_dir = join(home_dir, "bin")
+    elif not IS_WIN:
+        lib_dir = join(home_dir, "lib", PY_VERSION)
+        inc_dir = join(home_dir, "include", PY_VERSION + ABI_FLAGS)
+        bin_dir = join(home_dir, "bin")
+    return home_dir, lib_dir, inc_dir, bin_dir
+
+def main():
+    args = sys.argv[1:]
+    if not args:
+        print("You must provide a DEST_DIR")
+        sys.exit(2)
+    if len(args) > 1:
+        print("There must be only one argument: DEST_DIR (you gave {})".format(" ".join(args)))
+        sys.exit(2)
+
+    home_dir = args[0]
+
+    if os.path.exists(home_dir) and os.path.isfile(home_dir):
+        logger.fatal("ERROR: File already exists and is not a directory.")
+        logger.fatal("Please provide a different path or delete the file.")
+        sys.exit(3)
+
+    if os.pathsep in home_dir:
+        logger.fatal("ERROR: target path contains the operating system path separator '{}'".format(os.pathsep))
+        logger.fatal("This is not allowed as would make the activation scripts unusable.".format(os.pathsep))
+        sys.exit(3)
+
+    if os.environ.get("WORKING_ENV"):
+        logger.fatal("ERROR: you cannot run virtualenv while in a working env")
+        logger.fatal("Please deactivate your working env, then re-run this script")
+        sys.exit(3)
+
+    make_environment_relocatable(home_dir)
+
+if __name__ == "__main__":
+    main()
diff --git a/src/tools/ivigenerator/deploy-virtualenv-files.txt b/src/tools/ivigenerator/deploy-virtualenv-files.txt
index 4874cd7..5808a1e 100644
--- a/src/tools/ivigenerator/deploy-virtualenv-files.txt
+++ b/src/tools/ivigenerator/deploy-virtualenv-files.txt
@@ -1,3 +1,36 @@
+codecs.py
+io.py
+site.py
+os.py
+stat.py
+ntpath.py
+genericpath.py
+re.py
+types.py
+sre_*.py
+functools.py
+operator.py
+keyword.py
+heapq.py
+reprlib.py
+copyreg.py
+weakref.py
+struct.py
+warnings.py
+linecache.py
+tokenize.py
+token.py
+copy.py
+base64.py
+fnmatch.py
+posixpath.py
+shutil.py
+tempfile.py
+random.py
+bisect.py
+contextvars.py
+locale.py
+hashlib.py
 LICENSE.txt
 _*.py
 abc.py
diff --git a/src/tools/ivigenerator/deploy-virtualenv.bat b/src/tools/ivigenerator/deploy-virtualenv.bat
index 9be976d..8997b3d 100644
--- a/src/tools/ivigenerator/deploy-virtualenv.bat
+++ b/src/tools/ivigenerator/deploy-virtualenv.bat
@@ -28,6 +28,7 @@
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 
 SET SCRIPT=%~dp0
+SETLOCAL ENABLEDELAYEDEXPANSION
 
 IF %1%=="" (
     call:usage
@@ -51,12 +52,24 @@ FOR %%F in (%VIRTUALENV_LIB%\python*) DO (
     )
 )
 
-IF NOT EXIST %LIB_FOLDER%\orig-prefix.txt (
-    echo "orig-prefix.txt doesn't exist";
-    exit 1
+IF EXIST %LIB_FOLDER%\orig-prefix.txt (
+    SET /p ORIG_PREFIX=<%LIB_FOLDER%\orig-prefix.txt
+) else (
+    IF EXIST "%VIRTUALENV%\pyvenv.cfg" (
+        FOR /f "tokens=1,2 delims==" %%a in (%VIRTUALENV%\pyvenv.cfg) DO (
+            SET NAME=%%a
+            SET NAME=!NAME:~0,-1!
+            IF !NAME!==base-prefix (
+                SET ORIG_PREFIX=%%b
+                SET ORIG_PREFIX=!ORIG_PREFIX:~1!
+            )
+        )
+    ) ELSE (
+        echo "Neither %LIB_FOLDER%\orig-prefix.txt nor %VIRTUALENV%\pyvenv.cfg exists"
+        exit 1
+    )
 )
 
-SET /p ORIG_PREFIX=<%LIB_FOLDER%\orig-prefix.txt
 SET ORIG_LIB=%ORIG_PREFIX%\lib\%PYTHON_VERSION%
 IF NOT EXIST "%ORIG_LIB%" (
     echo "%ORIG_LIB% doesn't exist"
diff --git a/src/tools/ivigenerator/deploy-virtualenv.sh b/src/tools/ivigenerator/deploy-virtualenv.sh
index b3efc8a..1aa792f 100755
--- a/src/tools/ivigenerator/deploy-virtualenv.sh
+++ b/src/tools/ivigenerator/deploy-virtualenv.sh
@@ -49,9 +49,15 @@ for file in "$VIRTUALENV_LIB"/python* ; do
     fi
 done
 [ ! -d "$LIB_FOLDER" ] && usage
-if [[ ! -e "$LIB_FOLDER/orig-prefix.txt" ]] ; then
-    echo "orig-prefix.txt doesn't exist";
-    exit 1
+if [[ -e "$LIB_FOLDER/orig-prefix.txt" ]] ; then
+    ORIG_PREFIX=$(<"$LIB_FOLDER"/orig-prefix.txt)
+else
+    if [[  -e "$VIRTUALENV/pyvenv.cfg" ]] ; then
+        ORIG_PREFIX=$(awk -F " = " '/base-prefix/ {print $2}' $VIRTUALENV/pyvenv.cfg)
+    else
+        echo "Neither $LIB_FOLDER/orig-prefix.txt or $VIRTUALENV/pyvenv.cfg exists";
+        exit 1
+    fi
 fi
 
 if [ "$(uname)" == "Darwin" ]; then
@@ -72,7 +78,6 @@ fi
 
 # Find all the locations used for the system python files
 # They are located in prefix, but we don't know the sub-folder (it is lib on most systems, but lib64 on some others)
-ORIG_PREFIX=$(<"$LIB_FOLDER"/orig-prefix.txt)
 ORIG_LIBS=`$VIRTUALENV/bin/python3 -c "import sys; print ('\n'.join(path for path in sys.path))" | grep $ORIG_PREFIX`
 
 if [[ ! -e "$SCRIPT/deploy-virtualenv-files.txt" ]] ; then
@@ -86,7 +91,7 @@ for ORIG_LIB in ${ORIG_LIBS} ; do
         for file in ${FILES} ; do
             expand_wildcard=($ORIG_LIB/$file)
             [ ! -e "$expand_wildcard" ] && continue;
-            cp -RLf "$ORIG_LIB"/$file "$LIB_FOLDER/"
+            cp -RLfn "$ORIG_LIB"/$file "$LIB_FOLDER/"
         done
 done
 
diff --git a/src/tools/ivigenerator/qface_internal_build.pri b/src/tools/ivigenerator/qface_internal_build.pri
index 99b84a7..e810120 100644
--- a/src/tools/ivigenerator/qface_internal_build.pri
+++ b/src/tools/ivigenerator/qface_internal_build.pri
@@ -14,6 +14,8 @@ VIRTUALENV_EXE = $$shell_quote($$QMAKE_PYTHON3_LOCATION) -m virtualenv
 # by adding -p we enforce that the python3 interpreter is used and make sure python3 is installed in the virtualenv
 VIRTUALENV_EXE += " -p $$shell_quote($$QMAKE_PYTHON3_LOCATION)"
 
+RELOCATE_VIRTUALENV = $$system_path($$QTIVI_ROOT/src/3rdparty/virtualenv/relocate_virtualenv.py)
+
 # Use a Python virtualenv for installing qface, so we don't pollute the user environment
 # On some systems virtualenv --always-copy doesn't work (https://github.com/pypa/virtualenv/issues/565).
 # To workaround the problem, we need to manually create the folder and create the virtualenv from
@@ -23,7 +25,7 @@ else:  qtivi_qface_virtualenv.target = qtivi_qface_virtualenv/bin/python
 qtivi_qface_virtualenv.commands = \
     $(MKDIR) qtivi_qface_virtualenv $$escape_expand(\n\t) \
     cd qtivi_qface_virtualenv && $$VIRTUALENV_EXE --always-copy . $$escape_expand(\n\t) \
-    cd qtivi_qface_virtualenv && $$VIRTUALENV_EXE --relocatable . $$escape_expand(\n\t) \
+    cd qtivi_qface_virtualenv && $$system_quote($$QMAKE_PYTHON3_LOCATION) $$RELOCATE_VIRTUALENV . $$escape_expand(\n\t) \
     @echo "Set up virtualenv for qface, name: qtivi_qface_virtualenv"
 QMAKE_EXTRA_TARGETS += qtivi_qface_virtualenv