cmake: add dependencies to rebuild libshiboken embedded files

It was often a pain to handle embedded files correctly. Often, when
embedding was used, one could not see any effect after
re-compiling when --reuse-build was specified.
This rendered the --reuse-build option often quite useless.

The new approach adds dependencies for all involved Python
files, and at the same time reduces them to a single list.

Fixes: PYSIDE-1536
Change-Id: I6ef36c9ffccd2cd1ac290bb8e879b050f633eaf4
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
(cherry picked from commit e641c225d2954cfc5c85e00eda83f5319c84b3c8)
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

4 files changed, 76 insertions, 42 deletions

diff --git a/sources/shiboken6/CMakeLists.txt b/sources/shiboken6/CMakeLists.txt
index 698128a23..5d3f4d439 100644
--- a/sources/shiboken6/CMakeLists.txt
+++ b/sources/shiboken6/CMakeLists.txt
@@ -190,6 +190,28 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
 endif()
 
 ######################################################################
+## Define the Python files involved in the build process.
+##
+## They are installed into the file system (see shibokenmodule)
+## and embedded into the libshiboken binary through a .zip file.
+######################################################################
+
+set(shiboken_python_files
+    "signature/lib/__init__.py"
+    "signature/lib/enum_sig.py"
+    "signature/lib/tool.py"
+    "signature/__init__.py"
+    "signature/errorhandler.py"
+    "signature/importhandler.py"
+    "signature/layout.py"
+    "signature/loader.py"
+    "signature/mapping.py"
+    "signature/parser.py"
+    "__init__.py"
+    "feature.py"
+    )
+
+######################################################################
 # Adding sub directories to build
 ######################################################################
 add_subdirectory(ApiExtractor)
diff --git a/sources/shiboken6/libshiboken/CMakeLists.txt b/sources/shiboken6/libshiboken/CMakeLists.txt
index 444fd8ce2..84355e4d6 100644
--- a/sources/shiboken6/libshiboken/CMakeLists.txt
+++ b/sources/shiboken6/libshiboken/CMakeLists.txt
@@ -29,13 +29,28 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/sbkversion.h.in"
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/embed/signature_bootstrap.py"
                "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap.py" @ONLY)
 
+# Variable from enclosing scope.
+list(TRANSFORM shiboken_python_files
+     PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/../shibokenmodule/files.dir/shibokensupport/"
+     OUTPUT_VARIABLE embedded_shiboken_files)
+
+if (QUIET_BUILD)
+    set(embedding_option "--quiet")
+else()
+    set(embedding_option "")
+endif()
+
 add_custom_command(
     OUTPUT  "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap_inc.h"
     OUTPUT  "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_inc.h"
     COMMAND ${PYTHON_EXECUTABLE} -E
             "${CMAKE_CURRENT_SOURCE_DIR}/embed/embedding_generator.py"
             --cmake-dir "${CMAKE_CURRENT_BINARY_DIR}/embed"
-            --limited-api ${PYTHON_LIMITED_API})
+            --limited-api ${PYTHON_LIMITED_API}
+            ${embedding_option}
+    DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/embed/embedding_generator.py"
+            ${embedded_shiboken_files}
+    )
 
 set(libshiboken_MAJOR_VERSION ${shiboken_MAJOR_VERSION})
 set(libshiboken_MINOR_VERSION ${shiboken_MINOR_VERSION})
diff --git a/sources/shiboken6/libshiboken/embed/embedding_generator.py b/sources/shiboken6/libshiboken/embed/embedding_generator.py
index a88da8d66..421d08a08 100644
--- a/sources/shiboken6/libshiboken/embed/embedding_generator.py
+++ b/sources/shiboken6/libshiboken/embed/embedding_generator.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2019 The Qt Company Ltd.
+## Copyright (C) 2021 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of PySide6.
@@ -60,18 +60,19 @@ import tempfile
 import argparse
 import marshal
 import traceback
+from pathlib import Path
 
-# work_dir is set to the source for testing, onl.
+# work_dir is set to the source for testing, only.
 # It can be overridden in the command line.
-work_dir = os.path.abspath(os.path.dirname(__file__))
+work_dir = Path(__file__).parent.resolve()
 embed_dir = work_dir
-cur_dir = os.getcwd()
-source_dir = os.path.normpath(os.path.join(work_dir, "..", "..", ".."))
-assert os.path.basename(source_dir) == "sources"
-build_script_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..", ".."))
-assert os.path.exists(os.path.join(build_script_dir, "build_scripts"))
+cur_dir = Path.cwd()
+source_dir = work_dir.parents[2]
+assert source_dir.name == "sources"
+build_script_dir = work_dir.parents[3]
+assert (build_script_dir / "build_scripts").exists()
 
-sys.path.insert(0, build_script_dir)
+sys.path.insert(0, os.fspath(build_script_dir))
 
 from build_scripts import utils
 
@@ -80,7 +81,7 @@ def runpy(cmd, **kw):
     subprocess.call([sys.executable, '-E'] + cmd.split(), **kw)
 
 
-def create_zipfile(limited_api):
+def create_zipfile(limited_api, quiet):
     """
     Collect all Python files, compile them, create a zip file
     and make a chunked base64 encoded file from it.
@@ -94,21 +95,20 @@ def create_zipfile(limited_api):
     # Note that we could improve that with the PyZipfile function to use .pyc files
     # in different folders, but that makes only sense when COIN allows us to have
     # multiple Python versions in parallel.
-    from os.path import join, getsize
     for root, dirs, files in os.walk(work_dir):
         for name in files:
-            fpath = os.path.join(root, name)
+            fpath = Path(root) / name
             ew = name.endswith
             if ew(".pyc") or ew(".pyo") or ew(".zip") or ew(".inc"):
                 os.remove(fpath)
     # We copy every Python file into this dir, but only for the right version.
     # For testing in the source dir, we need to filter.
     ignore = []
-    utils.copydir(os.path.join(source_dir, "shiboken6", "shibokenmodule", "files.dir", "shibokensupport"),
-                  os.path.join(work_dir, "shibokensupport"),
+    utils.copydir(source_dir / "shiboken6" / "shibokenmodule" / "files.dir" / "shibokensupport",
+                  work_dir / "shibokensupport",
                   ignore=ignore, file_filter_function=lambda name, n2: name.endswith(".py"))
     if embed_dir != work_dir:
-        utils.copyfile(os.path.join(embed_dir, "signature_bootstrap.py"), work_dir)
+        utils.copyfile(embed_dir / "signature_bootstrap.py", work_dir)
 
     if limited_api:
         pass   # We cannot compile, unless we have folders per Python version
@@ -124,11 +124,26 @@ def create_zipfile(limited_api):
     with open(inc_name, "w") as inc:
         _embed_file(tmp, inc)
     tmp.close()
+
     # also generate a simple embeddable .pyc file for signature_bootstrap.pyc
     boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc"
     with open(boot_name, "rb") as ldr, open("signature_bootstrap_inc.h", "w") as inc:
         _embed_bytefile(ldr, inc, limited_api)
     os.chdir(cur_dir)
+    if quiet:
+        return
+
+    # have a look at our populated folder unless quiet option
+    def tree(directory):
+        print(f'+ {directory}')
+        for path in sorted(directory.rglob('*')):
+            depth = len(path.relative_to(directory).parts)
+            spacer = '    ' * depth
+            print(f'{spacer}+ {path.name}')
+
+    print("++++ Current contents of")
+    tree(work_dir)
+    print("++++")
 
 
 def _embed_file(fin, fout):
@@ -233,7 +248,8 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('--cmake-dir', nargs="?")
     parser.add_argument('--limited-api', type=str2bool)
+    parser.add_argument('--quiet', action='store_true')
     args = parser.parse_args()
     if args.cmake_dir:
-        work_dir = os.path.abspath(args.cmake_dir)
-    create_zipfile(args.limited_api)
+        work_dir = Path(args.cmake_dir).resolve()
+    create_zipfile(args.limited_api, args.quiet)
diff --git a/sources/shiboken6/shibokenmodule/CMakeLists.txt b/sources/shiboken6/shibokenmodule/CMakeLists.txt
index 9ec90e46f..92fba52eb 100644
--- a/sources/shiboken6/shibokenmodule/CMakeLists.txt
+++ b/sources/shiboken6/shibokenmodule/CMakeLists.txt
@@ -42,30 +42,11 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/_config.py"
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in"
                "${CMAKE_CURRENT_BINARY_DIR}/__init__.py" @ONLY)
 
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/feature.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/feature.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/__init__.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/__init__.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/__init__.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/__init__.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/errorhandler.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/errorhandler.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/layout.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/layout.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/loader.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/loader.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/importhandler.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/importhandler.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/mapping.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/mapping.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/parser.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/parser.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/lib/__init__.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/lib/__init__.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/lib/enum_sig.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/lib/enum_sig.py" COPYONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/lib/tool.py"
-               "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/lib/tool.py" COPYONLY)
+# Variable from enclosing scope.
+foreach(item IN LISTS shiboken_python_files)
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/${item}"
+                   "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/${item}" COPYONLY)
+endforeach()
 
 install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/files.dir"
         DESTINATION "${PYTHON_SITE_PACKAGES}/shiboken6")