From afc3ba9023e7eae7a9df36932f68f5b66f120fcd Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Wed, 15 May 2019 12:49:45 +0200 Subject: generate_pyi: Simplify, Enhance and Get Rid Of Any Import Problems The generate_pyi.py script always had the problem that it was not easy to distinguish different kinds of ImportError. When ImportError was raised during a build, we assumed it was just a not-yet built module and ignored it. When patch 97df448e "Modernize cmake build" was checked in on 2019-02-07, a real import error was introduced on Windows. It was not recognized until 2019-05-13 that Windows had stopped generating .pyi files. It was discovered by working on an enhancement to dependency checks that erroneously looked for "libshiboken*" on windows. This should have raised an error in "generate_pyi.py" but did not, because the generation was skipped due to the real ImportError. General Redesign ---------------- It turned out that all the former difficulties with importing modules could be completely avoided, by explicitly using the inherent dependencies. The script was drastically simplified by using the module name as an argument. It was not clear in the first place that this would work, but actually we recognized that all modules a script wants to import are already built when CMake starts the .pyi generation. The only visible quirk is the pair QtMultimedia/QtMultimediaWidgets where both modules must be compiled before generating. (maybe that is an error in our XML files, or a Qt "feature"?) Enhancements ------------ The generate_pyi script is now completely deterministic, because all imports are enforced to work. There is no more polling for translated modules necessary. Everything is generated after a module was linked, The "--skip" semantic was first enhanced much further. In the end it was recognized that we don't need the parameter any longer, because with the determinism we are never computing a pyi file more than once. The parameter was then completely removed. The "--check" option was added for Python 3. It takes some time and is only automatically active in a COIN build. Task-number: PYSIDE-735 Change-Id: I3cc58f6cad80d8208e17f62d472fd48aa6aeebd6 Reviewed-by: Friedemann Kleint --- sources/pyside2/PySide2/support/generate_pyi.py | 211 +++++++---------------- sources/pyside2/cmake/Macros/PySideModules.cmake | 2 +- 2 files changed, 60 insertions(+), 153 deletions(-) diff --git a/sources/pyside2/PySide2/support/generate_pyi.py b/sources/pyside2/PySide2/support/generate_pyi.py index 377a53331..294cdc91b 100644 --- a/sources/pyside2/PySide2/support/generate_pyi.py +++ b/sources/pyside2/PySide2/support/generate_pyi.py @@ -1,7 +1,7 @@ # This Python file uses the following encoding: utf-8 ############################################################################# ## -## Copyright (C) 2018 The Qt Company Ltd. +## Copyright (C) 2019 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of Qt for Python. @@ -52,16 +52,9 @@ import io import re import subprocess import argparse -import glob -import math from contextlib import contextmanager from textwrap import dedent -import traceback - - import logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("generate_pyi") # Make sure not to get .pyc in Python2. @@ -73,6 +66,10 @@ USE_PEP563 = sys.version_info[:2] >= (3, 7) indent = " " * 4 is_py3 = sys.version_info[0] == 3 is_ci = os.environ.get("QTEST_ENVIRONMENT", "") == "ci" +is_debug = is_ci or os.environ.get("QTEST_ENVIRONMENT") + +logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO) +logger = logging.getLogger("generate_pyi") class Writer(object): @@ -118,7 +115,7 @@ class Formatter(Writer): self.print("import shiboken2 as Shiboken") self.print("Shiboken.Object = Object") self.print() - # This line will be replaced by the missing imports. + # This line will be replaced by the missing imports postprocess. self.print("IMPORTS") yield @@ -184,83 +181,36 @@ def find_imports(text): return [imp for imp in PySide2.__all__ if imp + "." in text] -_cache = {} - -def check_if_skipable(outfilepath): - # A file can be skipped if it exists, and if it's file time is not - # older than this script or any of its dependencies. - def _do_find_newest_module(): - newest = 0 - for obj in sys.modules.values(): - if getattr(obj, "__file__", None) and os.path.isfile(obj.__file__): - sourcepath = os.path.splitext(obj.__file__)[0] + ".py" - if os.path.exists(sourcepath): - newest = max(os.path.getmtime(sourcepath), newest) - return newest - - def find_newest_module(): - cache_name = "newest_module" - if cache_name not in _cache: - _cache[cache_name] = _do_find_newest_module() - return _cache[cache_name] - - if os.path.exists(outfilepath): - stamp = os.path.getmtime(outfilepath) - if stamp >= find_newest_module(): - return True - return False - - def generate_pyi(import_name, outpath, options): """ Generates a .pyi file. - - Returns 1 If the result is valid, -1 if the result existed already - and was skipped, else 0. - - This function will get called during a PySide build, and many concurrent - process might try to create .pyi files. We let only one process at a - time work on these files, but it will still be different processes which - do the work. """ - pid = os.getpid() plainname = import_name.split(".")[-1] outfilepath = os.path.join(outpath, plainname + ".pyi") - if options.skip and check_if_skipable(outfilepath): - logger.debug("{pid}:Skipped existing: {op}" - .format(op=os.path.basename(outfilepath), **locals())) - return -1 - - try: - top = __import__(import_name) - obj = getattr(top, plainname) - if not getattr(obj, "__file__", None) or os.path.isdir(obj.__file__): - raise ImportError("We do not accept a namespace as module {plainname}" - .format(**locals())) - module = sys.modules[import_name] - - outfile = io.StringIO() - fmt = Formatter(outfile) - enu = HintingEnumerator(fmt) - fmt.print(get_license_text()) # which has encoding, already - need_imports = not USE_PEP563 - if USE_PEP563: - fmt.print("from __future__ import annotations") - fmt.print() - fmt.print(dedent('''\ - """ - This file contains the exact signatures for all functions in module - {import_name}, except for defaults which are replaced by "...". - """ - '''.format(**locals()))) - enu.module(import_name) + top = __import__(import_name) + obj = getattr(top, plainname) + if not getattr(obj, "__file__", None) or os.path.isdir(obj.__file__): + raise ModuleNotFoundError("We do not accept a namespace as module " + "{plainname}".format(**locals())) + module = sys.modules[import_name] + + outfile = io.StringIO() + fmt = Formatter(outfile) + fmt.print(get_license_text()) # which has encoding, already + need_imports = not USE_PEP563 + if USE_PEP563: + fmt.print("from __future__ import annotations") fmt.print() - fmt.print("# eof") - - except ImportError as e: - logger.debug("{pid}:Import problem with module {plainname}: {e}".format(**locals())) - return 0 - + fmt.print(dedent('''\ + """ + This file contains the exact signatures for all functions in module + {import_name}, except for defaults which are replaced by "...". + """ + '''.format(**locals()))) + HintingEnumerator(fmt).module(import_name) + fmt.print() + fmt.print("# eof") + # Postprocess: resolve the imports with open(outfilepath, "w") as realfile: wr = Writer(realfile) outfile.seek(0) @@ -282,23 +232,9 @@ def generate_pyi(import_name, outpath, options): else: wr.print(line) logger.info("Generated: {outfilepath}".format(**locals())) - if is_py3: + if is_py3 and (options.check or is_ci): # Python 3: We can check the file directly if the syntax is ok. subprocess.check_output([sys.executable, outfilepath]) - return 1 - - -@contextmanager -def single_process(lockdir): - try: - os.mkdir(lockdir) - try: - yield lockdir - finally: - # make sure to cleanup, even if we leave with CTRL-C - os.rmdir(lockdir) - except OSError: - yield None def generate_all_pyi(outpath, options): @@ -316,69 +252,40 @@ def generate_all_pyi(outpath, options): from PySide2.support.signature import inspect from PySide2.support.signature.lib.enum_sig import HintingEnumerator - valid = check = 0 - if not outpath: - outpath = os.path.dirname(PySide2.__file__) - lockdir = os.path.join(outpath, "generate_pyi.lockdir") - - pyi_var = "GENERATE_PYI_RECURSE {}".format(math.pi) # should not be set by anybody - if not os.environ.get(pyi_var, ""): - # To catch a possible crash, we run as a subprocess: - os.environ[pyi_var] = "yes" - ret = subprocess.call([sys.executable] + sys.argv) - if ret and os.path.exists(lockdir): - os.rmdir(lockdir) - sys.exit(ret) - # We are the subprocess. Do the real work. - with single_process(lockdir) as locked: - if locked: - if is_ci: - # When COIN is running, we sometimes get racing conditions with - # the windows manifest tool which wants access to a module that - # we already have imported. But when we wait until all binaries - # are created, that cannot happen, because we are then the last - # process, and the tool has already been run. - bin_pattern = "Qt*.pyd" if sys.platform == "win32" else "Qt*.so" - search = os.path.join(PySide2.__path__[0], bin_pattern) - if len(glob.glob(search)) < len(PySide2.__all__): - return - for mod_name in PySide2.__all__: - import_name = "PySide2." + mod_name - step = generate_pyi(import_name, outpath, options) - valid += abs(step) - check += step - - npyi = len(PySide2.__all__) - # Prevent too many messages when '--reuse-build' is used. We check that - # all files are created, but at least one was really computed. - if valid == npyi and check != -npyi: - logger.info("+++ All {npyi} .pyi files have been created.".format(**locals())) + outpath = outpath or os.path.dirname(PySide2.__file__) + name_list = PySide2.__all__ if options.modules == ["all"] else options.modules + errors = ", ".join(set(name_list) - set(PySide2.__all__)) + if errors: + raise ImportError("The module(s) '{errors}' do not exist".format(**locals())) + quirk1, quirk2 = "QtMultimedia", "QtMultimediaWidgets" + if name_list == [quirk1]: + logger.debug("Note: We must defer building of {quirk1}.pyi until {quirk2} " + "is available".format(**locals())) + name_list = [] + elif name_list == [quirk2]: + name_list = [quirk1, quirk2] + for mod_name in name_list: + import_name = "PySide2." + mod_name + generate_pyi(import_name, outpath, options) if __name__ == "__main__": - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command") - # create the parser for the "run" command - parser_run = subparsers.add_parser("run", - help="run the generation", + parser = argparse.ArgumentParser( description="This script generates the .pyi file for all PySide modules.") - parser_run.add_argument("--skip", action="store_true", - help="skip existing files") - parser_run.add_argument("--quiet", action="store_true", help="Run quietly") - parser_run.add_argument("--outpath", + parser.add_argument("modules", nargs="+", + help="'all' or the names of modules to build (QtCore QtGui etc.)") + parser.add_argument("--quiet", action="store_true", help="Run quietly") + parser.add_argument("--check", action="store_true", help="Test the output if on Python 3") + parser.add_argument("--outpath", help="the output directory (default = binary location)") - parser_run.add_argument("--sys-path", nargs="+", + parser.add_argument("--sys-path", nargs="+", help="a list of strings prepended to sys.path") options = parser.parse_args() - if options.command == "run": - if options.quiet: - logger.setLevel(logging.WARNING) - outpath = options.outpath - if outpath and not os.path.exists(outpath): - os.makedirs(outpath) - logger.info("+++ Created path {outpath}".format(**locals())) - generate_all_pyi(outpath, options=options) - else: - parser_run.print_help() - sys.exit(1) + if options.quiet: + logger.setLevel(logging.WARNING) + outpath = options.outpath + if outpath and not os.path.exists(outpath): + os.makedirs(outpath) + logger.info("+++ Created path {outpath}".format(**locals())) + generate_all_pyi(outpath, options=options) # eof diff --git a/sources/pyside2/cmake/Macros/PySideModules.cmake b/sources/pyside2/cmake/Macros/PySideModules.cmake index 77dc8c8ac..67d27cb81 100644 --- a/sources/pyside2/cmake/Macros/PySideModules.cmake +++ b/sources/pyside2/cmake/Macros/PySideModules.cmake @@ -182,7 +182,7 @@ macro(create_pyside_module) make_path(path_value "${path_value}") string(APPEND ld_prefix "${PATH_SEP}${path_value}") endif() - set(generate_pyi_options run --skip --sys-path + set(generate_pyi_options ${module_NAME} --sys-path "${pysidebindings_BINARY_DIR}" "${SHIBOKEN_PYTHON_MODULE_DIR}") if (QUIET_BUILD) -- cgit v1.2.3 From ff3a9f61edc479e5610f6d57c5ab789704ca81cb Mon Sep 17 00:00:00 2001 From: Alexandru Croitor Date: Tue, 21 May 2019 11:33:16 +0200 Subject: Fix propagation of existing LD env vars when calling generate_pyi Existing PATH, LD_LIBRARIES_PATH or DYLD_LIBRARY_PATH values should be appended to the ones we set ourselves. This takes care of finding any system libraries which a distro / OS might set by default. Change-Id: I1b3d1ea20c024e8ffcba3fb38d9088917130abfe Fixes: PYSIDE-1008 Reviewed-by: Friedemann Kleint Reviewed-by: Christian Tismer --- sources/pyside2/cmake/Macros/PySideModules.cmake | 29 ++++++++++++++++-------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/sources/pyside2/cmake/Macros/PySideModules.cmake b/sources/pyside2/cmake/Macros/PySideModules.cmake index 67d27cb81..dca00ec11 100644 --- a/sources/pyside2/cmake/Macros/PySideModules.cmake +++ b/sources/pyside2/cmake/Macros/PySideModules.cmake @@ -162,25 +162,36 @@ macro(create_pyside_module) # Need to set the LD_ env vars before invoking the script, because it might use build-time # libraries instead of install time libraries. if (WIN32) - set(ld_prefix "PATH=") + set(ld_prefix_var_name "PATH") elseif(APPLE) - set(ld_prefix "DYLD_LIBRARY_PATH=") + set(ld_prefix_var_name "DYLD_LIBRARY_PATH") else() - set(ld_prefix "LD_LIBRARY_PATH=") + set(ld_prefix_var_name "LD_LIBRARY_PATH") endif() - set(ld_prefix "${ld_prefix}${pysidebindings_BINARY_DIR}/libpyside${PATH_SEP}${SHIBOKEN_SHARED_LIBRARY_DIR}") + set(ld_prefix "${ld_prefix_var_name}=${pysidebindings_BINARY_DIR}/libpyside${PATH_SEP}${SHIBOKEN_SHARED_LIBRARY_DIR}") - # On Windows we also need to propagate the whole environment PATH value, because pyside modules - # import Qt, and the Qt modules are found from PATH. + # Append any existing ld_prefix values, so existing PATH, LD_LIBRARY_PATH, etc. + # On Windows it is needed because pyside modules import Qt, + # and the Qt modules are found from PATH. + # On Linux and macOS, existing values might be set to find system libraries correctly. + # For example on openSUSE when compiling with icc, libimf.so from Intel has to be found. if(WIN32) # Get the value of PATH with CMake separators. - file(TO_CMAKE_PATH "$ENV{PATH}" path_value) + file(TO_CMAKE_PATH "$ENV{${ld_prefix_var_name}}" path_value) # Replace the CMake list separators with "\;"s, to avoid the PATH values being # interpreted as CMake list elements, we actually want to pass the whole string separated # by ";" to the command line. - make_path(path_value "${path_value}") - string(APPEND ld_prefix "${PATH_SEP}${path_value}") + if(path_value) + make_path(path_value "${path_value}") + string(APPEND ld_prefix "${PATH_SEP}${path_value}") + endif() + else() + # Handles both macOS and Linux. + set(env_value "$ENV{${ld_prefix_var_name}}") + if(env_value) + string(APPEND ld_prefix ":${env_value}") + endif() endif() set(generate_pyi_options ${module_NAME} --sys-path "${pysidebindings_BINARY_DIR}" -- cgit v1.2.3 From 91cb27a2a7180e9c05b806d870b266fa31bef041 Mon Sep 17 00:00:00 2001 From: Friedemann Kleint Date: Tue, 30 Oct 2018 11:47:02 +0100 Subject: shiboken: Make it possible to extend namespaces Qt has some modules, where functions and classes are added to namespaces of other modules. For example, QtGui adds the free functions mightBeRichText() and convertFromPlainText() to the "Qt" namespace and Qt3DQuick adds a namespace "Quick" to the namespace "Qt3DCore" started in Qt3DCore. Shiboken was unable to handle this since the name of the index constant was derived from the namespace name and would thus clash. Also, all code model items from the base module would be re-generated into the extended namespace. To fix this: - No longer join namespace code model items in clang builder. - Search for pre-existing namespaces in traverseNamespace() before creating a new one, continuing populating it. - Add a "files" attribute taking a regular expression to match header names to the namespace type entry, making it possible to restrict code model items by files when extending namespaces (as otherwise, all base items are again added). - Add "extends" attribute specifying the package name of the namespace to extend, which will be used as base type in CPython. - Disambiguate the SBK indexes of namespaces and the init_ functions by adding the module name. Change-Id: Ib552e878911942fa4d52d2bb0c6695e5b2c4c341 Reviewed-by: Cristian Maureira-Fredes --- sources/pyside2/PySide2/QtGui/CMakeLists.txt | 1 + .../PySide2/QtGui/typesystem_gui_common.xml | 5 ++ sources/pyside2/tests/QtGui/CMakeLists.txt | 1 + .../pyside2/tests/QtGui/qtextdocument_functions.py | 47 +++++++++++++++ .../shiboken2/ApiExtractor/abstractmetabuilder.cpp | 56 ++++++++++++++---- .../shiboken2/ApiExtractor/abstractmetabuilder_p.h | 6 +- sources/shiboken2/ApiExtractor/abstractmetalang.h | 8 +++ .../ApiExtractor/clangparser/clangbuilder.cpp | 14 ++--- sources/shiboken2/ApiExtractor/messages.cpp | 29 +++++++++- sources/shiboken2/ApiExtractor/messages.h | 9 +++ .../shiboken2/ApiExtractor/parser/codemodel.cpp | 16 ++++++ sources/shiboken2/ApiExtractor/parser/codemodel.h | 4 ++ sources/shiboken2/ApiExtractor/typedatabase.cpp | 34 +++++++++-- sources/shiboken2/ApiExtractor/typedatabase.h | 3 +- .../shiboken2/ApiExtractor/typedatabase_typedefs.h | 2 + sources/shiboken2/ApiExtractor/typesystem.cpp | 66 ++++++++++++++++++++-- sources/shiboken2/ApiExtractor/typesystem.h | 20 ++++++- sources/shiboken2/ApiExtractor/typesystem_p.h | 6 ++ .../shiboken2/generator/shiboken2/cppgenerator.cpp | 12 +++- .../generator/shiboken2/shibokengenerator.cpp | 10 +++- 20 files changed, 309 insertions(+), 40 deletions(-) create mode 100644 sources/pyside2/tests/QtGui/qtextdocument_functions.py diff --git a/sources/pyside2/PySide2/QtGui/CMakeLists.txt b/sources/pyside2/PySide2/QtGui/CMakeLists.txt index 48354987d..18d80f647 100644 --- a/sources/pyside2/PySide2/QtGui/CMakeLists.txt +++ b/sources/pyside2/PySide2/QtGui/CMakeLists.txt @@ -197,6 +197,7 @@ ${QtGui_GEN_DIR}/qtouchdevice_wrapper.cpp ${QtGui_GEN_DIR}/qtouchevent_touchpoint_wrapper.cpp ${QtGui_GEN_DIR}/qtouchevent_wrapper.cpp ${QtGui_GEN_DIR}/qtransform_wrapper.cpp +${QtGui_GEN_DIR}/qt_wrapper.cpp ${QtGui_GEN_DIR}/qvalidator_wrapper.cpp ${QtGui_GEN_DIR}/qvector2d_wrapper.cpp ${QtGui_GEN_DIR}/qvector3d_wrapper.cpp diff --git a/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml b/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml index 71001140c..67270ea67 100644 --- a/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml +++ b/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml @@ -202,6 +202,11 @@ + + + diff --git a/sources/pyside2/tests/QtGui/CMakeLists.txt b/sources/pyside2/tests/QtGui/CMakeLists.txt index 31747659e..927e72468 100644 --- a/sources/pyside2/tests/QtGui/CMakeLists.txt +++ b/sources/pyside2/tests/QtGui/CMakeLists.txt @@ -40,6 +40,7 @@ PYSIDE_TEST(qrasterwindow_test.py) PYSIDE_TEST(qopenglwindow_test.py) PYSIDE_TEST(qregion_test.py) PYSIDE_TEST(qstylehints_test.py) +PYSIDE_TEST(qtextdocument_functions.py) PYSIDE_TEST(qtextdocument_undoredo_test.py) PYSIDE_TEST(qtextdocumentwriter_test.py) PYSIDE_TEST(qtextline_test.py) diff --git a/sources/pyside2/tests/QtGui/qtextdocument_functions.py b/sources/pyside2/tests/QtGui/qtextdocument_functions.py new file mode 100644 index 000000000..f1376aa5b --- /dev/null +++ b/sources/pyside2/tests/QtGui/qtextdocument_functions.py @@ -0,0 +1,47 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the test suite of Qt for Python. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import unittest + +from PySide2.QtGui import Qt +from PySide2.QtCore import QTextCodec + + +class QTextDocumentFunctions(unittest.TestCase): + + def testFunctions(self): + self.assertFalse(Qt.mightBeRichText('bla')) + self.assertTrue(Qt.mightBeRichText('

bla

')) + html = Qt.convertFromPlainText("A & B", Qt.WhiteSpaceNormal) + self.assertEqual(html, '

A & B

') + codec = Qt.codecForHtml(b'bla') + self.assertTrue(codec) + + +if __name__ == '__main__': + unittest.main() diff --git a/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp b/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp index 7e998d315..67489b151 100644 --- a/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp +++ b/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp @@ -452,10 +452,8 @@ void AbstractMetaBuilderPrivate::traverseDom(const FileModelItem &dom) const auto &namespaceTypeValues = dom->namespaces(); ReportHandler::startProgress("Generating namespace model (" + QByteArray::number(namespaceTypeValues.size()) + ")..."); - for (const NamespaceModelItem &item : namespaceTypeValues) { - if (AbstractMetaClass *metaClass = traverseNamespace(dom, item)) - addAbstractMetaClass(metaClass, item.data()); - } + for (const NamespaceModelItem &item : namespaceTypeValues) + traverseNamespace(dom, item); // Go through all typedefs to see if we have defined any // specific typedefs to be used as classes. @@ -742,23 +740,38 @@ AbstractMetaClass *AbstractMetaBuilderPrivate::traverseNamespace(const FileModel if (!namespaceName.isEmpty()) namespaceName.append(colonColon()); namespaceName.append(namespaceItem->name()); - NamespaceTypeEntry *type = TypeDatabase::instance()->findNamespaceType(namespaceName); if (TypeDatabase::instance()->isClassRejected(namespaceName)) { m_rejectedClasses.insert(namespaceName, AbstractMetaBuilder::GenerationDisabled); return 0; } + auto type = TypeDatabase::instance()->findNamespaceType(namespaceName, namespaceItem->fileName()); if (!type) { qCWarning(lcShiboken).noquote().nospace() << QStringLiteral("namespace '%1' does not have a type entry").arg(namespaceName); return 0; } - AbstractMetaClass* metaClass = new AbstractMetaClass; - metaClass->setTypeEntry(type); - - *metaClass += AbstractMetaAttributes::Public; + // Continue populating namespace? + AbstractMetaClass *metaClass = AbstractMetaClass::findClass(m_metaClasses, type); + if (!metaClass) { + metaClass = new AbstractMetaClass; + metaClass->setTypeEntry(type); + *metaClass += AbstractMetaAttributes::Public; + addAbstractMetaClass(metaClass, namespaceItem.data()); + if (auto extendsType = type->extends()) { + AbstractMetaClass *extended = AbstractMetaClass::findClass(m_metaClasses, extendsType); + if (!extended) { + qCWarning(lcShiboken, "%s", + qPrintable(msgNamespaceToBeExtendedNotFound(extendsType->name(), extendsType->targetLangPackage()))); + return nullptr; + } + metaClass->setExtendedNamespace(extended); + } + } else { + m_itemToClass.insert(namespaceItem.data(), metaClass); + } if (ReportHandler::isDebug(ReportHandler::SparseDebug)) { qCDebug(lcShiboken) @@ -797,7 +810,6 @@ AbstractMetaClass *AbstractMetaBuilderPrivate::traverseNamespace(const FileModel if (mjc) { metaClass->addInnerClass(mjc); mjc->setEnclosingClass(metaClass); - addAbstractMetaClass(mjc, ni.data()); } } @@ -3075,6 +3087,30 @@ AbstractMetaClassList AbstractMetaBuilderPrivate::classesTopologicalSorted(const return result; } +void AbstractMetaBuilderPrivate::pushScope(const NamespaceModelItem &item) +{ + // For purposes of type lookup, join all namespaces of the same name + // within the parent item. + QVector candidates; + const QString name = item->name(); + if (!m_scopes.isEmpty()) { + for (const auto &n : m_scopes.constLast()->namespaces()) { + if (n->name() == name) + candidates.append(n); + } + } + if (candidates.size() > 1) { + NamespaceModelItem joined(new _NamespaceModelItem(m_scopes.constLast()->model(), + name, _CodeModelItem::Kind_Namespace)); + joined->setScope(item->scope()); + for (const auto &n : candidates) + joined->appendNamespace(*n); + m_scopes << joined; + } else { + m_scopes << item; + } +} + AbstractMetaClassList AbstractMetaBuilder::classesTopologicalSorted(const AbstractMetaClassList &classList, const Dependencies &additionalDependencies) const { diff --git a/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h b/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h index 3c0039f0e..1fd5f3c34 100644 --- a/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h +++ b/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h @@ -56,9 +56,9 @@ public: const Dependencies &additionalDependencies = Dependencies()) const; ScopeModelItem popScope() { return m_scopes.takeLast(); } - void pushScope(ScopeModelItem item) { m_scopes << item; } + void pushScope(const NamespaceModelItem &item); - ScopeModelItem currentScope() const { return m_scopes.constLast(); } + NamespaceModelItem currentScope() const { return m_scopes.constLast(); } AbstractMetaClass *argumentToClass(const ArgumentModelItem &, AbstractMetaClass *currentClass); @@ -182,7 +182,7 @@ public: QHash m_enums; - QList m_scopes; + QList m_scopes; QSet m_setupInheritanceDone; diff --git a/sources/shiboken2/ApiExtractor/abstractmetalang.h b/sources/shiboken2/ApiExtractor/abstractmetalang.h index ef4cef2b4..e8ec21f48 100644 --- a/sources/shiboken2/ApiExtractor/abstractmetalang.h +++ b/sources/shiboken2/ApiExtractor/abstractmetalang.h @@ -1486,6 +1486,12 @@ public: return m_enclosingClass; } + /** + * \return the namespace from another package which this namespace extends. + */ + AbstractMetaClass *extendedNamespace() const { return m_extendedNamespace; } + void setExtendedNamespace(AbstractMetaClass *e) { m_extendedNamespace = e; } + void setEnclosingClass(AbstractMetaClass *cl) { m_enclosingClass = cl; @@ -1729,6 +1735,8 @@ private: const AbstractMetaClass *m_enclosingClass = nullptr; AbstractMetaClass *m_baseClass = nullptr; + AbstractMetaClass *m_extendedNamespace = nullptr; + const AbstractMetaClass *m_templateBaseClass = nullptr; AbstractMetaFunctionList m_functions; AbstractMetaFieldList m_fields; diff --git a/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp b/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp index 40f915028..3ced0e06c 100644 --- a/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp +++ b/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp @@ -885,15 +885,13 @@ BaseVisitor::StartTokenResult Builder::startToken(const CXCursor &cursor) appendDiagnostic(d); return Error; } - // If possible, continue existing namespace (as otherwise, all headers - // where a namespace is continued show up in the type database). + // Treat namespaces separately to allow for extending namespaces + // in subsequent modules. NamespaceModelItem namespaceItem = parentNamespaceItem->findNamespace(name); - if (namespaceItem.isNull()) { - namespaceItem.reset(new _NamespaceModelItem(d->m_model, name)); - setFileName(cursor, namespaceItem.data()); - namespaceItem->setScope(d->m_scope); - parentNamespaceItem->addNamespace(namespaceItem); - } + namespaceItem.reset(new _NamespaceModelItem(d->m_model, name)); + setFileName(cursor, namespaceItem.data()); + namespaceItem->setScope(d->m_scope); + parentNamespaceItem->addNamespace(namespaceItem); d->pushScope(namespaceItem); } break; diff --git a/sources/shiboken2/ApiExtractor/messages.cpp b/sources/shiboken2/ApiExtractor/messages.cpp index fdae2359b..a6e75aac3 100644 --- a/sources/shiboken2/ApiExtractor/messages.cpp +++ b/sources/shiboken2/ApiExtractor/messages.cpp @@ -206,8 +206,6 @@ QString msgCannotTranslateTemplateArgument(int i, return result; } -// abstractmetalang.cpp - QString msgDisallowThread(const AbstractMetaFunction *f) { QString result; @@ -219,6 +217,13 @@ QString msgDisallowThread(const AbstractMetaFunction *f) return result; } +QString msgNamespaceToBeExtendedNotFound(const QString &namespaceName, const QString &packageName) +{ + return QLatin1String("The namespace '") + namespaceName + + QLatin1String("' to be extended cannot be found in package ") + + packageName + QLatin1Char('.'); +} + // docparser.cpp QString msgCannotFindDocumentation(const QString &fileName, @@ -421,6 +426,26 @@ QString msgRejectReason(const TypeRejection &r, const QString &needle) return result; } +// typesystem.cpp + +QString msgCannotFindNamespaceToExtend(const QString &name, + const QStringRef &extendsPackage) +{ + return QLatin1String("Cannot find namespace ") + name + + QLatin1String(" in package ") + extendsPackage; +} + +QString msgExtendingNamespaceRequiresPattern(const QString &name) +{ + return QLatin1String("Namespace ") + name + + QLatin1String(" requires a file pattern since it extends another namespace."); +} + +QString msgInvalidRegularExpression(const QString &pattern, const QString &why) +{ + return QLatin1String("Invalid pattern \"") + pattern + QLatin1String("\": ") + why; +} + // qtdocgenerator.cpp QString msgTagWarning(const QXmlStreamReader &reader, const QString &context, diff --git a/sources/shiboken2/ApiExtractor/messages.h b/sources/shiboken2/ApiExtractor/messages.h index 30b13fbf8..ab2bf64b6 100644 --- a/sources/shiboken2/ApiExtractor/messages.h +++ b/sources/shiboken2/ApiExtractor/messages.h @@ -85,6 +85,8 @@ QString msgCannotTranslateTemplateArgument(int i, QString msgDisallowThread(const AbstractMetaFunction *f); +QString msgNamespaceToBeExtendedNotFound(const QString &namespaceName, const QString &packageName); + QString msgCannotFindDocumentation(const QString &fileName, const char *what, const QString &name, const QString &query); @@ -117,6 +119,13 @@ QString msgLeftOverArguments(const QMap &remainingArgs); QString msgInvalidVersion(const QString &package, const QString &version); +QString msgCannotFindNamespaceToExtend(const QString &name, + const QStringRef &extendsPackage); + +QString msgExtendingNamespaceRequiresPattern(const QString &name); + +QString msgInvalidRegularExpression(const QString &pattern, const QString &why); + QString msgCyclicDependency(const QString &funcName, const QString &graphName, const QVector &involvedConversions); diff --git a/sources/shiboken2/ApiExtractor/parser/codemodel.cpp b/sources/shiboken2/ApiExtractor/parser/codemodel.cpp index 9a845b04a..eb0f44689 100644 --- a/sources/shiboken2/ApiExtractor/parser/codemodel.cpp +++ b/sources/shiboken2/ApiExtractor/parser/codemodel.cpp @@ -773,6 +773,16 @@ void _ScopeModelItem::addEnum(EnumModelItem item) m_enums.append(item); } +void _ScopeModelItem::appendScope(const _ScopeModelItem &other) +{ + m_classes += other.m_classes; + m_enums += other.m_enums; + m_typeDefs += other.m_typeDefs; + m_variables += other.m_variables; + m_functions += other.m_functions; + m_enumsDeclarations += other.m_enumsDeclarations; +} + #ifndef QT_NO_DEBUG_STREAM template static void formatScopeHash(QDebug &d, const char *prefix, const Hash &h, @@ -899,6 +909,12 @@ _FileModelItem::~_FileModelItem() { } +void _NamespaceModelItem::appendNamespace(const _NamespaceModelItem &other) +{ + appendScope(other); + m_namespaces += other.m_namespaces; +} + #ifndef QT_NO_DEBUG_STREAM void _NamespaceModelItem::formatDebug(QDebug &d) const { diff --git a/sources/shiboken2/ApiExtractor/parser/codemodel.h b/sources/shiboken2/ApiExtractor/parser/codemodel.h index 64415e07b..3bce5e216 100644 --- a/sources/shiboken2/ApiExtractor/parser/codemodel.h +++ b/sources/shiboken2/ApiExtractor/parser/codemodel.h @@ -360,6 +360,8 @@ protected: explicit _ScopeModelItem(CodeModel *model, const QString &name, int kind = __node_kind) : _CodeModelItem(model, name, kind) {} + void appendScope(const _ScopeModelItem &other); + #ifndef QT_NO_DEBUG_STREAM void formatScopeItemsDebug(QDebug &d) const; #endif @@ -440,6 +442,8 @@ public: NamespaceModelItem findNamespace(const QString &name) const; + void appendNamespace(const _NamespaceModelItem &other); + #ifndef QT_NO_DEBUG_STREAM void formatDebug(QDebug &d) const override; #endif diff --git a/sources/shiboken2/ApiExtractor/typedatabase.cpp b/sources/shiboken2/ApiExtractor/typedatabase.cpp index a8c69d376..930f85d30 100644 --- a/sources/shiboken2/ApiExtractor/typedatabase.cpp +++ b/sources/shiboken2/ApiExtractor/typedatabase.cpp @@ -621,14 +621,33 @@ ObjectTypeEntry* TypeDatabase::findObjectType(const QString& name) const return 0; } -NamespaceTypeEntry* TypeDatabase::findNamespaceType(const QString& name) const +NamespaceTypeEntryList TypeDatabase::findNamespaceTypes(const QString& name) const { + NamespaceTypeEntryList result; const auto entries = findTypes(name); for (TypeEntry *entry : entries) { - if (entry->isNamespace() && useType(entry)) - return static_cast(entry); + if (entry->isNamespace()) + result.append(static_cast(entry)); } - return 0; + return result; +} + +NamespaceTypeEntry *TypeDatabase::findNamespaceType(const QString& name, + const QString &fileName) const +{ + const auto entries = findNamespaceTypes(name); + // Preferably check on matching file name first, if a pattern was given. + if (!fileName.isEmpty()) { + for (NamespaceTypeEntry *entry : entries) { + if (entry->hasPattern() && entry->matchesFile(fileName)) + return entry; + } + } + for (NamespaceTypeEntry *entry : entries) { + if (!entry->hasPattern()) + return entry; + } + return nullptr; } bool TypeDatabase::shouldDropTypeEntry(const QString& fullTypeName) const @@ -836,6 +855,13 @@ void EnumTypeEntry::formatDebug(QDebug &d) const d << ", flags=(" << m_flags << ')'; } +void NamespaceTypeEntry::formatDebug(QDebug &d) const +{ + ComplexTypeEntry::formatDebug(d); + auto pattern = m_filePattern.pattern(); + FORMAT_NONEMPTY_STRING("pattern", pattern) +} + void ContainerTypeEntry::formatDebug(QDebug &d) const { ComplexTypeEntry::formatDebug(d); diff --git a/sources/shiboken2/ApiExtractor/typedatabase.h b/sources/shiboken2/ApiExtractor/typedatabase.h index 0040364bf..7f1b2a3fc 100644 --- a/sources/shiboken2/ApiExtractor/typedatabase.h +++ b/sources/shiboken2/ApiExtractor/typedatabase.h @@ -87,7 +87,8 @@ public: PrimitiveTypeEntry* findPrimitiveType(const QString& name) const; ComplexTypeEntry* findComplexType(const QString& name) const; ObjectTypeEntry* findObjectType(const QString& name) const; - NamespaceTypeEntry* findNamespaceType(const QString& name) const; + NamespaceTypeEntryList findNamespaceTypes(const QString& name) const; + NamespaceTypeEntry *findNamespaceType(const QString& name, const QString &fileName = QString()) const; ContainerTypeEntry* findContainerType(const QString& name) const; FunctionTypeEntry* findFunctionType(const QString& name) const; const TypeSystemTypeEntry *findTypeSystemType(const QString &name) const; diff --git a/sources/shiboken2/ApiExtractor/typedatabase_typedefs.h b/sources/shiboken2/ApiExtractor/typedatabase_typedefs.h index fbbbabe43..f9591609e 100644 --- a/sources/shiboken2/ApiExtractor/typedatabase_typedefs.h +++ b/sources/shiboken2/ApiExtractor/typedatabase_typedefs.h @@ -34,6 +34,7 @@ #include class ContainerTypeEntry; +class NamespaceTypeEntry; class PrimitiveTypeEntry; class TemplateEntry; class TypeEntry; @@ -61,6 +62,7 @@ typedef QMap TypeEntryMap; typedef QMap TypedefEntryMap; typedef QVector ContainerTypeEntryList; +using NamespaceTypeEntryList = QVector; typedef QVector PrimitiveTypeEntryList; #endif // TYPEDATABASE_TYPEDEFS_H diff --git a/sources/shiboken2/ApiExtractor/typesystem.cpp b/sources/shiboken2/ApiExtractor/typesystem.cpp index 318a52e2e..434134be9 100644 --- a/sources/shiboken2/ApiExtractor/typesystem.cpp +++ b/sources/shiboken2/ApiExtractor/typesystem.cpp @@ -126,8 +126,7 @@ static bool setRejectionRegularExpression(const QString &patternIn, pattern = QLatin1Char('^') + QRegularExpression::escape(patternIn) + QLatin1Char('$'); re->setPattern(pattern); if (!re->isValid()) { - *errorMessage = QLatin1String("Invalid pattern \"") + patternIn - + QLatin1String("\": ") + re->errorString(); + *errorMessage = msgInvalidRegularExpression(patternIn, re->errorString()); return false; } return true; @@ -1286,6 +1285,47 @@ ObjectTypeEntry * return otype; } +NamespaceTypeEntry * + Handler::parseNamespaceTypeEntry(const QXmlStreamReader &reader, + const QString &name, const QVersionNumber &since, + QXmlStreamAttributes *attributes) +{ + QScopedPointer result(new NamespaceTypeEntry(name, since)); + applyCommonAttributes(result.data(), attributes); + applyComplexTypeAttributes(reader, result.data(), attributes); + for (int i = attributes->size() - 1; i >= 0; --i) { + const QStringRef attributeName = attributes->at(i).qualifiedName(); + if (attributeName == QLatin1String("files")) { + const QString pattern = attributes->takeAt(i).value().toString(); + QRegularExpression re(pattern); + if (!re.isValid()) { + m_error = msgInvalidRegularExpression(pattern, re.errorString()); + return nullptr; + } + result->setFilePattern(re); + } else if (attributeName == QLatin1String("extends")) { + const auto extendsPackageName = attributes->takeAt(i).value(); + auto allEntries = TypeDatabase::instance()->findNamespaceTypes(name); + auto extendsIt = std::find_if(allEntries.cbegin(), allEntries.cend(), + [extendsPackageName] (const NamespaceTypeEntry *e) { + return e->targetLangPackage() == extendsPackageName; + }); + if (extendsIt == allEntries.cend()) { + m_error = msgCannotFindNamespaceToExtend(name, extendsPackageName); + return nullptr; + } + result->setExtends(*extendsIt); + } + } + + if (result->extends() && !result->hasPattern()) { + m_error = msgExtendingNamespaceRequiresPattern(name); + return nullptr; + } + + return result.take(); +} + ValueTypeEntry * Handler::parseValueTypeEntry(const QXmlStreamReader &, const QString &name, const QVersionNumber &since, @@ -2631,7 +2671,7 @@ bool Handler::startElement(const QXmlStreamReader &reader) if (element->type != StackElement::PrimitiveTypeEntry && element->type != StackElement::FunctionTypeEntry) { TypeEntry *tmp = m_database->findType(name); - if (tmp) + if (tmp && !tmp->isNamespace()) qCWarning(lcShiboken).noquote().nospace() << QStringLiteral("Duplicate type entry: '%1'").arg(name); } @@ -2710,9 +2750,10 @@ bool Handler::startElement(const QXmlStreamReader &reader) } break; case StackElement::NamespaceTypeEntry: - element->entry = new NamespaceTypeEntry(name, since); - applyCommonAttributes(element->entry, &attributes); - applyComplexTypeAttributes(reader, static_cast(element->entry), &attributes); + if (auto entry = parseNamespaceTypeEntry(reader, name, since, &attributes)) + element->entry = entry; + else + return false; break; case StackElement::ObjectTypeEntry: element->entry = new ObjectTypeEntry(name, since); @@ -3799,8 +3840,21 @@ TypeEntry *NamespaceTypeEntry::clone() const return new NamespaceTypeEntry(*this); } +void NamespaceTypeEntry::setFilePattern(const QRegularExpression &r) +{ + m_filePattern = r; + m_hasPattern = !m_filePattern.pattern().isEmpty(); + if (m_hasPattern) + m_filePattern.optimize(); +} + NamespaceTypeEntry::NamespaceTypeEntry(const NamespaceTypeEntry &) = default; +bool NamespaceTypeEntry::matchesFile(const QString &needle) const +{ + return m_filePattern.match(needle).hasMatch(); +} + ValueTypeEntry::ValueTypeEntry(const QString &name, const QVersionNumber &vr) : ComplexTypeEntry(name, BasicValueType, vr) { diff --git a/sources/shiboken2/ApiExtractor/typesystem.h b/sources/shiboken2/ApiExtractor/typesystem.h index f089bb6e0..2e4578a1d 100644 --- a/sources/shiboken2/ApiExtractor/typesystem.h +++ b/sources/shiboken2/ApiExtractor/typesystem.h @@ -1531,10 +1531,28 @@ public: TypeEntry *clone() const override; + const NamespaceTypeEntry *extends() const { return m_extends; } + void setExtends(const NamespaceTypeEntry *e) { m_extends = e; } + + const QRegularExpression &filePattern() const { return m_filePattern; } // restrict files + void setFilePattern(const QRegularExpression &r); + + bool hasPattern() const { return m_hasPattern; } + + bool matchesFile(const QString &needle) const; + +#ifndef QT_NO_DEBUG_STREAM + void formatDebug(QDebug &d) const override; +#endif + protected: NamespaceTypeEntry(const NamespaceTypeEntry &); -}; +private: + QRegularExpression m_filePattern; + const NamespaceTypeEntry *m_extends = nullptr; + bool m_hasPattern = false; +}; class ValueTypeEntry : public ComplexTypeEntry { diff --git a/sources/shiboken2/ApiExtractor/typesystem_p.h b/sources/shiboken2/ApiExtractor/typesystem_p.h index f6b0717f4..8a8fcb359 100644 --- a/sources/shiboken2/ApiExtractor/typesystem_p.h +++ b/sources/shiboken2/ApiExtractor/typesystem_p.h @@ -175,6 +175,12 @@ private: parseFlagsEntry(const QXmlStreamReader &, EnumTypeEntry *enumEntry, const QString &name, QString flagName, const QVersionNumber &since, QXmlStreamAttributes *); + + NamespaceTypeEntry * + parseNamespaceTypeEntry(const QXmlStreamReader &, + const QString &name, const QVersionNumber &since, + QXmlStreamAttributes *attributes); + ObjectTypeEntry * parseInterfaceTypeEntry(const QXmlStreamReader &, const QString &name, const QVersionNumber &since, QXmlStreamAttributes *); diff --git a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp index 58788d5ef..8ee0a9cf2 100644 --- a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp +++ b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp @@ -4892,7 +4892,11 @@ void CppGenerator::writeFlagsUnaryOperator(QTextStream& s, const AbstractMetaEnu QString CppGenerator::getSimpleClassInitFunctionName(const AbstractMetaClass *metaClass) const { - QString initFunctionName = metaClass->qualifiedCppName(); + QString initFunctionName; + // Disambiguate namespaces per module to allow for extending them. + if (metaClass->isNamespace()) + initFunctionName += moduleName(); + initFunctionName += metaClass->qualifiedCppName(); initFunctionName.replace(QLatin1String("::"), QLatin1String("_")); return initFunctionName; } @@ -4996,9 +5000,11 @@ void CppGenerator::writeClassRegister(QTextStream &s, } // 7:baseType - if (metaClass->baseClass()) { + const auto base = metaClass->isNamespace() + ? metaClass->extendedNamespace() : metaClass->baseClass(); + if (base) { s << INDENT << "reinterpret_cast(" - << cpythonTypeNameExt(metaClass->baseClass()->typeEntry()) << ")," << endl; + << cpythonTypeNameExt(base->typeEntry()) << ")," << endl; } else { s << INDENT << "0," << endl; } diff --git a/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp b/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp index 44405c700..2b3b20c75 100644 --- a/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp +++ b/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp @@ -2667,8 +2667,14 @@ QString ShibokenGenerator::getTypeIndexVariableName(const TypeEntry* type) if (trueType->basicReferencedTypeEntry()) type = trueType->basicReferencedTypeEntry(); } - QString result = QLatin1String("SBK_") - + _fixedCppTypeName(type->qualifiedCppName()).toUpper(); + QString result = QLatin1String("SBK_"); + // Disambiguate namespaces per module to allow for extending them. + if (type->isNamespace()) { + QString package = type->targetLangPackage(); + const int dot = package.lastIndexOf(QLatin1Char('.')); + result += package.rightRef(package.size() - (dot + 1)); + } + result += _fixedCppTypeName(type->qualifiedCppName()).toUpper(); appendIndexSuffix(&result); return result; } -- cgit v1.2.3