shiboken: optionally de-virtualize the Python files

The Python files in Shiboken are completely virtualized. That
means: There is no interaction with the external world that
could create interferences.

For debugging purposes or to modify the embedded files, it
is anyway useful to have an option to materialize these files
in the file system, again.

We add an environment variable `SBK_EMBED` that explicitly
unpacks the Python files (SBK_EMBED=0) or explicitly removes
the unpacked files (SBK_EMBED=1).

Without this variable, the state of embedding remains the same.

Change-Id: I1081b0f910482fb4b80a02f72c6bcce060db38e6
Fixes: PYSIDE-1994
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
(cherry picked from commit ee35c071ed8493cc5e3a80f19d9e2c2348160d56)
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

2 files changed, 82 insertions, 2 deletions

diff --git a/sources/pyside6/doc/developer/extras.rst b/sources/pyside6/doc/developer/extras.rst
index e172e05d7..79fc72190 100644
--- a/sources/pyside6/doc/developer/extras.rst
+++ b/sources/pyside6/doc/developer/extras.rst
@@ -23,3 +23,20 @@ Build on the command line
 =========================
 
 - Consider using ``build_scripts/qp5_tool.py``.
+
+De-Virtualize the Python Files
+==============================
+
+The Python files in the Shiboken module are completely virtual, i.E.
+they are nowhere existent in the file system for security reasons.
+
+For debugging purposes or to change something, it might be desirable
+to move these files into the normal file system, again.
+
+- Setting the environment variable "SBK_EMBED" once to false unpacks these
+  files when PySide6 or shiboken6 are imported. The files are written
+  into "side-packages/shiboken6/files.dir" and are used from then on.
+
+- Setting the variable to true removes "files.dir".
+
+- Without the "SBK_EMBED" variable, the embedding status remains sticky.
diff --git a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py
index c11a0367a..e9a49bfff 100644
--- a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py
+++ b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py
@@ -26,6 +26,7 @@ recursion_trap = 0
 import base64
 import importlib
 import io
+import os
 import sys
 import traceback
 import zipfile
@@ -35,6 +36,8 @@ from importlib.machinery import ModuleSpec
 from pathlib import Path
 
 
+sbks = "shibokensupport"
+
 def bootstrap():
 
     global recursion_trap
@@ -51,7 +54,6 @@ def bootstrap():
         # PYSIDE-1621: support_path can also be a finder instance.
         target.insert(0, support_path)
 
-        sbks = "shibokensupport"
         if sbks in sys.modules:
             del sys.modules[sbks]
         prefix = sbks + "."
@@ -71,11 +73,72 @@ def bootstrap():
             sys.exit(-1)
         target.remove(support_path)
 
-    target, support_path = prepare_zipfile()
+    # Here we decide if re we-incarnate the embedded files or use embedding.
+    incarnated = find_incarnated_files()
+    if incarnated:
+        target, support_path = sys.path, os.fspath(incarnated)
+    else:
+        target, support_path = prepare_zipfile()
     with ensure_shibokensupport(target, support_path):
         from shibokensupport.signature import loader
     return loader
 
+# Newer functionality:
+# This function checks if the support directory exist and returns it.
+# If does not exist, we try to create it and return it.
+# Otherwise, we return None.
+
+def find_incarnated_files():
+    import shiboken6 as root
+    files_dir = Path(root.__file__).resolve().parent / "files.dir"
+    handle_embedding_switch(files_dir)
+    if files_dir.exists():
+        sys.path.insert(0, os.fspath(files_dir))
+        # Note: To avoid recursion problems, we need to preload the loader.
+        # But that has the side-effect that we need to delay the feature
+        # initialization until all function pointers are set.
+        # See `post_init_func` in signature_globals.cpp .
+        import shibokensupport.signature.loader
+        del sys.path[0]
+        return files_dir
+    return None
+
+
+def handle_embedding_switch(files_dir):
+    """
+    This handles the optional environment variable `SBK_EMBED`
+    if not set             : do nothing
+    if set to 0, false, no : de-virtualize the Python files
+    if set to 1, true, yes : virtualize again (delete "files.dir")
+    """
+    env_name = "SBK_EMBED"
+    env_var = os.environ.get(env_name)
+    if not env_var:
+        return
+    if env_var.lower() in ("1", "t", "true", "y", "yes"):
+        import shutil
+        shutil.rmtree(files_dir, ignore_errors=True)
+    elif env_var.lower() in ("0", "f", "false", "n", "no"):
+        reincarnate_files(files_dir)
+
+
+def reincarnate_files(files_dir):
+    target, zip = prepare_zipfile()
+    names = (_ for _ in zip.zfile.namelist() if _.endswith(".py"))
+    try:
+        # First check mkdir to get an error when we cannot write.
+        files_dir.mkdir(exist_ok=True)
+    except os.error as e:
+        print(f"SBK_EMBED=False: Warning: Cannot write into {files_dir}")
+        return None
+    try:
+        # Then check for a real error when unpacking the zip file.
+        zip.zfile.extractall(path=files_dir, members=names)
+        return files_dir
+    except Exception as e:
+        print('Exception:', e)
+        traceback.print_exc(file=sys.stdout)
+        raise
 
 # New functionality: Loading from a zip archive.
 # There exists the zip importer, but as it is written, only real zip files are