Avoid too much stickiness when using --reuse-build

We had some unwanted cache effects and needed to manually
remove certain files before building. Otherwise it could happen
that a build pretended to be ok, although there was a bug that
prevented generation of the ".pyi" files.

Further investigation showed:
Using option "--reuse-build" with "no" and then with "yes" creates errors
in the signature module. This makes "reuse-build" useless in this case.
We now add an "a" to "pyside3d_build" as "pside3da_build" if
"--limited-api=yes" was given. (different proposals welcome.)
That solved most of the stickiness problems.

A left-over lock directory is removed now, since it would prevent
re-computation of the .pyi files. This is implemented by a recursive
call to the script, where the subprocess does the work and the
main process checks if there was a crash and removes the lock.

The "--skip" parameter of generate_pyi.py was refined:
When set, it is checked if the time stamp of all imported modules is
less than the ".pyi" file time stamp. Only then the generation is skipped.
By editing any involved python file, the ".pyi" files will be regenerated.

Task-number: PYSIDE-560
Change-Id: I1b1d8ffbc58db3d4b38bf65e3795efcad7e7870c
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>

3 files changed, 48 insertions, 7 deletions

diff --git a/build_scripts/main.py b/build_scripts/main.py
index b9c83b939..4d9f95b14 100644
--- a/build_scripts/main.py
+++ b/build_scripts/main.py
@@ -234,9 +234,11 @@ def prefix():
         name = "pyside"
     name += str(sys.version_info[0])
     if OPTION_DEBUG:
-        name += 'd'
+        name += "d"
     if is_debug_python():
-        name += 'p'
+        name += "p"
+    if OPTION_LIMITED_API == "yes" and sys.version_info[0] == 3:
+        name += "a"
     return name
 
 # Initialize, pull and checkout submodules
diff --git a/sources/pyside2/PySide2/support/generate_pyi.py b/sources/pyside2/PySide2/support/generate_pyi.py
index f286e34f1..377a53331 100644
--- a/sources/pyside2/PySide2/support/generate_pyi.py
+++ b/sources/pyside2/PySide2/support/generate_pyi.py
@@ -53,6 +53,7 @@ import re
 import subprocess
 import argparse
 import glob
+import math
 from contextlib import contextmanager
 from textwrap import dedent
 import traceback
@@ -183,6 +184,33 @@ def find_imports(text):
     return [imp for imp in PySide2.__all__ if imp + "." in text]
 
 
+_cache = {}
+
+def check_if_skipable(outfilepath):
+    # A file can be skipped if it exists, and if it's file time is not
+    # older than this script or any of its dependencies.
+    def _do_find_newest_module():
+        newest = 0
+        for obj in sys.modules.values():
+            if getattr(obj, "__file__", None) and os.path.isfile(obj.__file__):
+                sourcepath = os.path.splitext(obj.__file__)[0] + ".py"
+                if os.path.exists(sourcepath):
+                    newest = max(os.path.getmtime(sourcepath), newest)
+        return newest
+
+    def find_newest_module():
+        cache_name = "newest_module"
+        if cache_name not in _cache:
+            _cache[cache_name] = _do_find_newest_module()
+        return _cache[cache_name]
+
+    if os.path.exists(outfilepath):
+        stamp = os.path.getmtime(outfilepath)
+        if stamp >= find_newest_module():
+            return True
+    return False
+
+
 def generate_pyi(import_name, outpath, options):
     """
     Generates a .pyi file.
@@ -198,7 +226,7 @@ def generate_pyi(import_name, outpath, options):
     pid = os.getpid()
     plainname = import_name.split(".")[-1]
     outfilepath = os.path.join(outpath, plainname + ".pyi")
-    if options.skip and os.path.exists(outfilepath):
+    if options.skip and check_if_skipable(outfilepath):
         logger.debug("{pid}:Skipped existing: {op}"
                      .format(op=os.path.basename(outfilepath), **locals()))
         return -1
@@ -291,7 +319,17 @@ def generate_all_pyi(outpath, options):
     valid = check = 0
     if not outpath:
         outpath = os.path.dirname(PySide2.__file__)
-    lockdir = os.path.join(outpath, "generate_pyi.lockfile")
+    lockdir = os.path.join(outpath, "generate_pyi.lockdir")
+
+    pyi_var = "GENERATE_PYI_RECURSE {}".format(math.pi)  # should not be set by anybody
+    if not os.environ.get(pyi_var, ""):
+        # To catch a possible crash, we run as a subprocess:
+        os.environ[pyi_var] = "yes"
+        ret = subprocess.call([sys.executable] + sys.argv)
+        if ret and os.path.exists(lockdir):
+            os.rmdir(lockdir)
+        sys.exit(ret)
+    # We are the subprocess. Do the real work.
     with single_process(lockdir) as locked:
         if locked:
             if is_ci:
@@ -343,3 +381,4 @@ if __name__ == "__main__":
     else:
         parser_run.print_help()
         sys.exit(1)
+# eof
diff --git a/sources/shiboken2/libshiboken/embed/embedding_generator.py b/sources/shiboken2/libshiboken/embed/embedding_generator.py
index b6bfb1467..77aa5c329 100644
--- a/sources/shiboken2/libshiboken/embed/embedding_generator.py
+++ b/sources/shiboken2/libshiboken/embed/embedding_generator.py
@@ -92,7 +92,7 @@ def create_zipfile(limited_api):
     flag = '-b' if sys.version_info >= (3,) else ''
     os.chdir(work_dir)
 
-    # Limited API: Remove all left-over py[co] files first, in case we use '--reuse-build'.
+    # Remove all left-over py[co] and other files first, in case we use '--reuse-build'.
     # Note that we could improve that with the PyZipfile function to use .pyc files
     # in different folders, but that makes only sense when COIN allows us to have
     # multiple Python versions in parallel.
@@ -100,9 +100,9 @@ def create_zipfile(limited_api):
     for root, dirs, files in os.walk(work_dir):
         for name in files:
             fpath = os.path.join(root, name)
-            if name.endswith(".pyc") or name.endswith(".pyo"):
+            ew = name.endswith
+            if ew(".pyc") or ew(".pyo") or ew(".zip") or ew(".inc"):
                 os.remove(fpath)
-
     # We copy every Python file into this dir, but only for the right version.
     # For testing in the source dir, we need to filter.
     if sys.version_info[0] == 3: