Turn generate_pyi into a general pyi_generator tool, finish

After the new tool has been created, we can now produce
a Shiboken.pyi file automatically and make the PySide pyi
files more complete.

The Shiboken internal objects are now published, and we
no longer need a fake Shiboken.Object .

We can continue here a bit, maybe in another commit.

Task-number: PYSIDE-1415
Change-Id: I9ba9336dbffa200ac519968519ee9381dd5cad84
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
(cherry picked from commit c4b077486f2f85ec15fa9f75b6a8e34ce976f180)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

11 files changed, 137 insertions, 18 deletions

diff --git a/build_scripts/config.py b/build_scripts/config.py
index bcbc0b6e4..d3b9a38b1 100644
--- a/build_scripts/config.py
+++ b/build_scripts/config.py
@@ -203,7 +203,8 @@ class Config(object):
             setup_kwargs['install_requires'] = [f"{self.shiboken_module_st_name}=={package_version}"]
             setup_kwargs['entry_points'] = {
                 'console_scripts': [
-                    f'{SHIBOKEN} = {package_name}.scripts.shiboken_tool:main'
+                    f'{SHIBOKEN} = {package_name}.scripts.shiboken_tool:main',
+                    f'{SHIBOKEN}-genpyi = {package_name}.scripts.shiboken_tool:genpyi',
                 ]
             }
 
@@ -217,6 +218,7 @@ class Config(object):
                     f'{PYSIDE}-rcc = {package_name}.scripts.pyside_tool:rcc',
                     f'{PYSIDE}-designer= {package_name}.scripts.pyside_tool:designer',
                     f'{PYSIDE}-lupdate = {package_name}.scripts.pyside_tool:main',
+                    f'{PYSIDE}-genpyi = {package_name}.scripts.pyside_tool:genpyi',
                 ]
             }
         self.setup_kwargs = setup_kwargs
diff --git a/sources/pyside-tools/pyside_tool.py b/sources/pyside-tools/pyside_tool.py
index 0e2275c06..6a47d3632 100644
--- a/sources/pyside-tools/pyside_tool.py
+++ b/sources/pyside-tools/pyside_tool.py
@@ -128,5 +128,13 @@ def designer():
         qt_tool_wrapper("designer", sys.argv[1:])
 
 
+def genpyi():
+    pyside_dir = Path(__file__).resolve().parents[1]
+    support = pyside_dir / "support"
+    cmd = support / "generate_pyi.py"
+    command = [sys.executable, os.fspath(cmd)] + sys.argv[1:]
+    sys.exit(subprocess.call(command))
+
+
 if __name__ == "__main__":
     main()
diff --git a/sources/pyside6/PySide6/support/generate_pyi.py b/sources/pyside6/PySide6/support/generate_pyi.py
index dd09f72d9..74ef2acd7 100644
--- a/sources/pyside6/PySide6/support/generate_pyi.py
+++ b/sources/pyside6/PySide6/support/generate_pyi.py
@@ -119,5 +119,6 @@ if __name__ == "__main__":
         os.makedirs(outpath)
         logger.info(f"+++ Created path {outpath}")
     options._pyside_call = True
+    options.logger = logger
     generate_all_pyi(outpath, options=options)
 # eof
diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp
index d33ec63e0..a467abbc6 100644
--- a/sources/shiboken6/libshiboken/basewrapper.cpp
+++ b/sources/shiboken6/libshiboken/basewrapper.cpp
@@ -261,6 +261,9 @@ static PyType_Spec SbkObject_Type_spec = {
     SbkObject_Type_slots,
 };
 
+static const char *SbkObject_SignatureStrings[] = {
+    "Shiboken.Object(self)",
+    nullptr}; // Sentinel
 
 SbkObjectType *SbkObject_TypeF(void)
 {
@@ -881,6 +884,17 @@ void init()
     shibokenAlreadInitialised = true;
 }
 
+// PYSIDE-1415: Publish Shiboken objects.
+void initSignature(PyObject *module)
+{
+    auto type = reinterpret_cast<PyTypeObject *>(SbkObject_TypeF());
+    if (InitSignatureStrings(type, SbkObject_SignatureStrings) < 0)
+        return;
+
+    Py_INCREF(SbkObject_TypeF());
+    PyModule_AddObject(module, "Object", reinterpret_cast<PyObject *>(SbkObject_TypeF()));
+}
+
 // setErrorAboutWrongArguments now gets overload info from the signature module.
 // Info can be nullptr and contains extra info.
 void setErrorAboutWrongArguments(PyObject *args, const char *funcName, PyObject *info)
diff --git a/sources/shiboken6/libshiboken/basewrapper.h b/sources/shiboken6/libshiboken/basewrapper.h
index 2f0c22e9f..38f0d93eb 100644
--- a/sources/shiboken6/libshiboken/basewrapper.h
+++ b/sources/shiboken6/libshiboken/basewrapper.h
@@ -157,6 +157,8 @@ namespace Shiboken
 */
 LIBSHIBOKEN_API void init();
 
+/// PYSIDE-1415: Publish Shiboken objects.
+LIBSHIBOKEN_API void initSignature(PyObject *module);
 
 /// Delete the class T allocated on \p cptr.
 template<typename T>
diff --git a/sources/shiboken6/shiboken_tool.py b/sources/shiboken6/shiboken_tool.py
index 40f35f3ef..e136794f7 100755
--- a/sources/shiboken6/shiboken_tool.py
+++ b/sources/shiboken6/shiboken_tool.py
@@ -43,6 +43,7 @@ import os
 import subprocess
 from pathlib import Path
 
+
 def main():
     # The tools listed as entrypoints in setup.py are copied to 'scripts/..'
     cmd = Path("..") / Path(sys.argv[0]).name
@@ -50,5 +51,14 @@ def main():
     command.extend(sys.argv[1:])
     sys.exit(subprocess.call(command))
 
+
+def genpyi():
+    shiboken_dir = Path(__file__).resolve().parents[2] / "shiboken6"
+    support = shiboken_dir / "files.dir" / "shibokensupport"
+    cmd = support / "signature" / "lib" / "pyi_generator.py"
+    command = [sys.executable, os.fspath(cmd)] + sys.argv[1:]
+    sys.exit(subprocess.call(command))
+
+
 if __name__ == "__main__":
      main()
diff --git a/sources/shiboken6/shibokenmodule/CMakeLists.txt b/sources/shiboken6/shibokenmodule/CMakeLists.txt
index 9228572e3..0431ba6f2 100644
--- a/sources/shiboken6/shibokenmodule/CMakeLists.txt
+++ b/sources/shiboken6/shibokenmodule/CMakeLists.txt
@@ -43,6 +43,11 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/_config.py"
 # PYSIDE-1497: This `..` is the crucial trick to unify the path location of `Shiboken`.
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in"
                "${CMAKE_CURRENT_BINARY_DIR}/../__init__.py" @ONLY)
+# PYSIDE-1415: Copy Shiboken.pyi into the target.
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/Shiboken.pyi"
+               "${CMAKE_CURRENT_BINARY_DIR}/../Shiboken.pyi" @ONLY)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/../Shiboken.pyi"
+        DESTINATION "${PYTHON_SITE_PACKAGES}/shiboken6")
 
 # Variable from enclosing scope.
 foreach(item IN LISTS shiboken_python_files)
diff --git a/sources/shiboken6/shibokenmodule/Shiboken.pyi b/sources/shiboken6/shibokenmodule/Shiboken.pyi
new file mode 100644
index 000000000..84b907aa0
--- /dev/null
+++ b/sources/shiboken6/shibokenmodule/Shiboken.pyi
@@ -0,0 +1,80 @@
+# This Python file uses the following encoding: utf-8
+#############################################################################
+##
+## Copyright (C) 2021 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+from __future__ import annotations
+
+"""
+This file contains the exact signatures for all functions in module
+Shiboken, except for defaults which are replaced by "...".
+"""
+
+# Module `Shiboken`
+
+import typing
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from shiboken6 import Shiboken
+from shibokensupport.signature.mapping import (
+    Virtual, Missing, Invalid, Default, Instance)
+
+
+class Enum(object):
+
+    def __init__(self, itemValue: int = ...) -> None: ...
+
+class Object(object):
+
+    def __init__(self) -> None: ...
+
+class VoidPtr(object): ...
+
+
+def _unpickle_enum(arg__1: object, arg__2: object) -> object: ...
+def createdByPython(arg__1: object) -> bool: ...
+def delete(arg__1: object) -> None: ...
+def dump(arg__1: object) -> object: ...
+def getAllValidWrappers() -> object: ...
+def getCppPointer(arg__1: object) -> object: ...
+def invalidate(arg__1: object) -> None: ...
+def isValid(arg__1: object) -> bool: ...
+def ownedByPython(arg__1: object) -> bool: ...
+def wrapInstance(arg__1: int, arg__2: type) -> object: ...
+
+
+# eof
diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
index 8249d6adf..a71ed63f9 100644
--- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
+++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
@@ -156,11 +156,6 @@ class ExactEnumerator(object):
                 if len(enums):
                     self.section()
             for subclass_name, subclass in subclasses:
-                if klass == subclass:
-                    # this is a side effect of the typing module for Python 2.7
-                    # via the "._gorg" property, which we can safely ignore.
-                    print(f"Warning: {class_name} points to itself via {subclass_name}, skipped!")
-                    continue
                 ret.update(self.klass(subclass_name, subclass))
                 self.fmt.class_name = class_name
             if len(subclasses):
diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py
index 2d7b5102d..e95b6e959 100644
--- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py
+++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py
@@ -71,7 +71,7 @@ is_ci = os.environ.get("QTEST_ENVIRONMENT", "") == "ci"
 is_debug = is_ci or os.environ.get("QTEST_ENVIRONMENT")
 
 logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO)
-logger = logging.getLogger("generate_pyi")
+logger = logging.getLogger("pyi_generator")
 
 
 class Writer(object):
@@ -83,8 +83,7 @@ class Writer(object):
         # controlling too much blank lines
         if self.outfile:
             if args == () or args == ("",):
-                # Python 2.7 glitch: Empty tuples have wrong encoding.
-                # But we use that to skip too many blank lines:
+                # We use that to skip too many blank lines:
                 if self.history[-2:] == [True, True]:
                     return
                 print("", file=self.outfile, **kw)
@@ -151,10 +150,6 @@ class Formatter(Writer):
             from shiboken6 import Shiboken
             from {support}.signature.mapping import (
                 Virtual, Missing, Invalid, Default, Instance)
-
-            class Object(object): pass
-
-            Shiboken.Object = Object
             """
         self.print(dedent(txt))
         # This line will be replaced by the missing imports postprocess.
@@ -213,7 +208,7 @@ def get_license_text():
 
 
 def find_imports(text):
-    return [imp for imp in PySide6.__all__ if imp + "." in text]
+    return [imp for imp in PySide6.__all__ if f"PySide6.{imp}." in text]
 
 
 def find_module(import_name, outpath, from_pyside):
@@ -229,7 +224,7 @@ def find_module(import_name, outpath, from_pyside):
     # we are alone in external module mode
     p = Path(import_name).resolve()
     if not p.exists():
-        raise ValueError(f"File {import_name} does not exist.")
+        raise ValueError(f"File {p} does not exist.")
     if not outpath:
         outpath = p.parent
     # temporarily add the path and do the import
@@ -285,18 +280,22 @@ def generate_pyi(import_name, outpath, options):
                         imp = "PySide6." + mod_name
                         if imp != import_name:
                             wr.print("import " + imp)
-                wr.print("import " + import_name)
+                # Do not import Shiboken which is handled already.
+                if import_name != "Shiboken":
+                    wr.print("import " + import_name)
                 wr.print()
                 wr.print()
             else:
                 wr.print(line)
-    logger.info(f"Generated: {outfilepath}")
+    if not options.quiet:
+        options.logger.info(f"Generated: {outfilepath}")
     if options and options.check or is_ci:
         # Python 3.7 and up: We can check the file directly if the syntax is ok.
         if USE_PEP563:
             subprocess.check_output([sys.executable, outfilepath])
 
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -310,9 +309,10 @@ if __name__ == "__main__":
             """))
     parser.add_argument("module",
         help="The full path name of an importable module binary (.pyd, .so)")
+    parser.add_argument("--quiet", action="store_true", help="Run quietly")
     parser.add_argument("--check", action="store_true", help="Test the output")
     parser.add_argument("--outpath",
-        help="the output directory (default = binary location)")
+        help="the output directory (default = location of module binary)")
     options = parser.parse_args()
     module = options.module
     outpath = options.outpath
@@ -320,6 +320,7 @@ if __name__ == "__main__":
         os.makedirs(outpath)
         logger.info(f"+++ Created path {outpath}")
     options._pyside_call = False
+    options.logger = logger
     generate_pyi(module, outpath, options=options)
 
 # eof
diff --git a/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml b/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml
index ca6f497d5..7ebc990e1 100644
--- a/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml
+++ b/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml
@@ -124,6 +124,7 @@
         PyModule_AddObject(module, "__version_info__", version);
         PyModule_AddStringConstant(module, "__version__", SHIBOKEN_VERSION);
 
+        Shiboken::initSignature(module);
         VoidPtr::addVoidPtrToModule(module);
     </inject-code>
 </typesystem>