diff options
22 files changed, 1029 insertions, 305 deletions
diff --git a/sources/pyside2/PySide2/__init__.py.in b/sources/pyside2/PySide2/__init__.py.in index 4548f89b7..d896ab603 100644 --- a/sources/pyside2/PySide2/__init__.py.in +++ b/sources/pyside2/PySide2/__init__.py.in @@ -19,9 +19,6 @@ def _setupQtDirectories(): # loads the libraries into the process memory beforehand, and # thus takes care of it for us. import shiboken2 - # We might be running from CTest and missing shiboken2.__init__, - # so here it is again: Help Python2 by pre-loading modules. - import sys, zipfile, base64, marshal, io # Trigger signature initialization. type.__signature__ diff --git a/sources/pyside2/PySide2/support/generate_pyi.py b/sources/pyside2/PySide2/support/generate_pyi.py index 8474fa732..3b4b3409a 100644 --- a/sources/pyside2/PySide2/support/generate_pyi.py +++ b/sources/pyside2/PySide2/support/generate_pyi.py @@ -107,14 +107,14 @@ class Formatter(Writer): self.mod_name = mod_name self.print("# Module", mod_name) self.print("import PySide2") - self.print("import shiboken2 as Shiboken") - from shibokensupport.signature import typing - self.print("from shibokensupport.signature import typing") - self.print("from shibokensupport.signature.mapping import (") + from PySide2.support.signature import typing + self.print("from PySide2.support.signature import typing") + self.print("from PySide2.support.signature.mapping import (") self.print(" Virtual, Missing, Invalid, Default, Instance)") self.print() self.print("class Object(object): pass") self.print() + self.print("import shiboken2 as Shiboken") self.print("Shiboken.Object = Object") self.print() # This line will be replaced by the missing imports. @@ -289,8 +289,8 @@ def generate_all_pyi(outpath, options): # now we can import global PySide2, inspect, HintingEnumerator import PySide2 - from shibokensupport.signature import inspect - from shibokensupport.signature.lib.enum_sig import HintingEnumerator + from PySide2.support.signature import inspect + from PySide2.support.signature.lib.enum_sig import HintingEnumerator valid = check = 0 if not outpath: diff --git a/sources/pyside2/tests/pysidetest/CMakeLists.txt b/sources/pyside2/tests/pysidetest/CMakeLists.txt index 1b1baf39a..cb8ba04cf 100644 --- a/sources/pyside2/tests/pysidetest/CMakeLists.txt +++ b/sources/pyside2/tests/pysidetest/CMakeLists.txt @@ -140,3 +140,4 @@ PYSIDE_TEST(mixin_signal_slots_test.py) PYSIDE_TEST(signal_slot_warning.py) PYSIDE_TEST(all_modules_load_test.py) PYSIDE_TEST(qapp_like_a_macro_test.py) +PYSIDE_TEST(embedding_test.py) diff --git a/sources/pyside2/tests/pysidetest/embedding_test.py b/sources/pyside2/tests/pysidetest/embedding_test.py new file mode 100644 index 000000000..aa71360ca --- /dev/null +++ b/sources/pyside2/tests/pysidetest/embedding_test.py @@ -0,0 +1,74 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import unittest + +# This test tests the embedding feature of PySide. +# Normally, embedding is only used when necessary. +# By setting the variable "pyside_uses_embedding", +# we enforce usage of embedding. + + +class EmbeddingTest(unittest.TestCase): + + # def test_pyside_normal(self): + # import sys + # self.assertFalse(hasattr(sys, "pyside_uses_embedding")) + # import PySide2 + # # everything has to be imported + # self.assertTrue("PySide2.support.signature" in sys.modules) + # # there should be a variale in sys, now (no idea if set) + # self.assertTrue(hasattr(sys, "pyside_uses_embedding")) + + # Unfortunately, I see no way how to shut things enough down + # to trigger a second initiatization. Therefore, only one test :-/ + def test_pyside_embedding(self): + import sys, os + self.assertFalse(hasattr(sys, "pyside_uses_embedding")) + sys.pyside_uses_embedding = "anything true" + import PySide2 + # everything has to be imported + self.assertTrue("PySide2.support.signature" in sys.modules) + self.assertEqual(sys.pyside_uses_embedding, True) + dn = os.path.dirname + name = os.path.basename(dn(dn(dn(PySide2.support.signature.__file__)))) + self.assertTrue(name.startswith("embedded.") and name.endswith(".zip")) + +if __name__ == '__main__': + unittest.main() diff --git a/sources/pyside2/tests/registry/existence_test.py b/sources/pyside2/tests/registry/existence_test.py index 83f9d79f9..62795f232 100644 --- a/sources/pyside2/tests/registry/existence_test.py +++ b/sources/pyside2/tests/registry/existence_test.py @@ -116,7 +116,9 @@ class TestSignaturesExists(unittest.TestCase): continue if key not in found_sigs: warn("missing key: '{}'".format(key)) - elif isinstance(value, list) and len(value) != len(found_sigs[key]): + elif isinstance(value, list) and len(value) > len(found_sigs[key]): + # We check that nothing got lost. But it is ok when an older + # registry file does not have all variants, yet! warn(msgMultiSignatureCount(key, found_sigs[key], value)) if is_ci and check_warnings(): raise RuntimeError("There are errors, see above.") @@ -132,7 +134,9 @@ class TestSignaturesExists(unittest.TestCase): continue if key not in found_sigs: warn("missing key: '{}'".format(key)) - elif isinstance(value, list) and len(value) != len(found_sigs[key]): + elif isinstance(value, list) and len(value) > len(found_sigs[key]): + # We check that nothing got lost. But it is ok when an older + # registry file does not have all variants, yet! warn(msgMultiSignatureCount(key, found_sigs[key], value)) self.assertTrue(check_warnings()) diff --git a/sources/shiboken2/libshiboken/CMakeLists.txt b/sources/shiboken2/libshiboken/CMakeLists.txt index 636d8c01b..7cbb22978 100644 --- a/sources/shiboken2/libshiboken/CMakeLists.txt +++ b/sources/shiboken2/libshiboken/CMakeLists.txt @@ -26,14 +26,16 @@ endif() configure_file("${CMAKE_CURRENT_SOURCE_DIR}/sbkversion.h.in" "${CMAKE_CURRENT_BINARY_DIR}/sbkversion.h" @ONLY) -# configure_file("${CMAKE_CURRENT_SOURCE_DIR}/embed/signature_loader.py" -# "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_loader.py" @ONLY) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/embed/signature_bootstrap.py" + "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap.py" @ONLY) -# add_custom_command( -# OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature.inc" -# COMMAND ${PYTHON_EXECUTABLE} -E -# "${CMAKE_CURRENT_SOURCE_DIR}/embed/embedding_generator.py" -# --cmake-dir "${CMAKE_CURRENT_BINARY_DIR}/embed") +add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap.inc" + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature.inc" + COMMAND ${PYTHON_EXECUTABLE} -E + "${CMAKE_CURRENT_SOURCE_DIR}/embed/embedding_generator.py" + --cmake-dir "${CMAKE_CURRENT_BINARY_DIR}/embed" + --limited-api ${PYTHON_LIMITED_API}) set(libshiboken_MAJOR_VERSION ${shiboken_MAJOR_VERSION}) set(libshiboken_MINOR_VERSION ${shiboken_MINOR_VERSION}) @@ -60,7 +62,8 @@ pep384impl.cpp voidptr.cpp typespec.cpp bufferprocs_py37.cpp -# embed/signature.inc +embed/signature_bootstrap.inc +embed/signature.inc ) get_numpy_location() diff --git a/sources/shiboken2/libshiboken/embed/embedding_generator.py b/sources/shiboken2/libshiboken/embed/embedding_generator.py new file mode 100644 index 000000000..3ee96a1a5 --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/embedding_generator.py @@ -0,0 +1,241 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +embedding_generator.py + +This file takes the content of the two supported directories and inserts +it into a zip file. The zip file is then converted into a C++ source +file that can easily be unpacked again with Python (see signature.cpp, +constant 'PySide_PythonCode'). + +Note that this _is_ a zipfile, but since it is embedded into the shiboken +binary, we cannot use the zipimport module from Python. +But a similar solution is possible that allows for normal imports. + +See signature_bootstrap.py for details. +""" + +from __future__ import print_function, absolute_import + +import sys +import os +import subprocess +import textwrap +import tempfile +import argparse +import marshal +import traceback + +# work_dir is set to the source for testing, onl. +# It can be overridden in the command line. +work_dir = os.path.abspath(os.path.dirname(__file__)) +embed_dir = work_dir +cur_dir = os.getcwd() +source_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..")) +assert os.path.basename(source_dir) == "sources" +build_script_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..", "..")) +assert os.path.exists(os.path.join(build_script_dir, "build_scripts")) + +sys.path.insert(0, build_script_dir) + +from build_scripts import utils + + +def runpy(cmd, **kw): + subprocess.call([sys.executable, '-E'] + cmd.split(), **kw) + + +def create_zipfile(limited_api): + """ + Collect all Python files, compile them, create a zip file + and make a chunked base64 encoded file from it. + """ + zip_name = "signature.zip" + inc_name = "signature.inc" + flag = '-b' if sys.version_info >= (3,) else '' + os.chdir(work_dir) + + # Limited API: Remove all left-over py[co] files first, in case we use '--reuse-build'. + # Note that we could improve that with the PyZipfile function to use .pyc files + # in different folders, but that makes only sense when COIN allows us to have + # multiple Python versions in parallel. + from os.path import join, getsize + for root, dirs, files in os.walk(work_dir): + for name in files: + fpath = os.path.join(root, name) + if name.endswith(".pyc") or name.endswith(".pyo"): + os.remove(fpath) + + # We copy every Python file into this dir, but only for the right version. + # For testing in the source dir, we need to filter. + if sys.version_info[0] == 3: + ignore = "backport_inspect.py typing27.py".split() + else: + ignore = "".split() + utils.copydir(os.path.join(source_dir, "shiboken2", "shibokenmodule", "files.dir", "shibokensupport"), + os.path.join(work_dir, "shibokensupport"), + ignore=ignore, file_filter_function=lambda name, n2: name.endswith(".py")) + if embed_dir != work_dir: + utils.copyfile(os.path.join(embed_dir, "signature_bootstrap.py"), work_dir) + + if limited_api: + pass # We cannot compile, unless we have folders per Python version + else: + files = ' '.join(fn for fn in os.listdir('.')) + runpy('-m compileall -q {flag} {files}'.format(**locals())) + files = ' '.join(fn for fn in os.listdir('.') if not fn == zip_name) + runpy('-m zipfile -c {zip_name} {files}'.format(**locals())) + tmp = tempfile.TemporaryFile(mode="w+") + runpy('-m base64 {zip_name}'.format(**locals()), stdout=tmp) + # now generate the include file + tmp.seek(0) + with open(inc_name, "w") as inc: + _embed_file(tmp, inc) + # also generate a simple embeddable .pyc file for signature_bootstrap.pyc + boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc" + with open(boot_name, "rb") as ldr, open("signature_bootstrap.inc", "w") as inc: + _embed_bytefile(ldr, inc, limited_api) + os.chdir(cur_dir) + + +def _embed_file(fin, fout): + """ + Format a text file for embedding in a C++ source file. + """ + # MSVC has a 64k string limitation. In C, it would be easy to create an + # array of 64 byte strings and use them as one big array. In C++ this does + # not work, since C++ insists in having the terminating nullbyte. + # Therefore, we split the string after an arbitrary number of lines + # (chunked file). + limit = 50 + text = fin.readlines() + print(textwrap.dedent(""" + /* + * This is a ZIP archive of all Python files in the directory + * "shiboken2/shibokenmodule/files.dir/shibokensupport/signature" + * There is also a toplevel file "signature_bootstrap.py[c]" that will be + * directly executed from C++ as a bootstrap loader. + */ + """).strip(), file=fout) + block, blocks = 0, len(text) // limit + 1 + for idx, line in enumerate(text): + if idx % limit == 0: + comma = "," if block else "" + block += 1 + print(file=fout) + print('/* Block {block} of {blocks} */{comma}'.format(**locals()), file=fout) + print('\"{}\"'.format(line.strip()), file=fout) + print('/* Sentinel */, \"\"', file=fout) + + +def _embed_bytefile(fin, fout, is_text): + """ + Format a binary file for embedding in a C++ source file. + This version works directly with a single .pyc file. + """ + fname = fin.name + remark = ("No .pyc file because '--LIMITED-API=yes'" if is_text else + "The .pyc header is stripped away") + print(textwrap.dedent(""" + /* + * This is the file "{fname}" as a simple byte array. + * It can be directly embedded without any further processing. + * {remark}. + */ + """).format(**locals()).strip(), file=fout) + headsize = ( 0 if is_text else + 16 if sys.version_info >= (3, 7) else 12 if sys.version_info >= (3, 3) else 8) + binstr = fin.read()[headsize:] + if is_text: + try: + compile(binstr, fin.name, "exec") + except SyntaxError as e: + print(e) + traceback.print_exc(file=sys.stdout) + print(textwrap.dedent(""" + ************************************************************************* + *** + *** Could not compile the boot loader '{fname}'! + *** + ************************************************************************* + """).format(version=sys.version_info[:3], **locals())) + raise SystemError + else: + try: + marshal.loads(binstr) + except ValueError as e: + print(e) + traceback.print_exc(file=sys.stdout) + print(textwrap.dedent(""" + ************************************************************************* + *** + *** This Python version {version} seems to have a new .pyc header size. + *** Please correct the 'headsize' constant ({headsize}). + *** + ************************************************************************* + """).format(version=sys.version_info[:3], **locals())) + raise SystemError + + print(file=fout) + use_ord = sys.version_info[0] == 2 + for i in range(0, len(binstr), 16): + for c in bytes(binstr[i : i + 16]): + print("{:#4},".format(ord(c) if use_ord else c), file=fout, end="") + print(file=fout) + print("/* End Of File */", file=fout) + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--cmake-dir', nargs="?") + parser.add_argument('--limited-api', type=str2bool) + args = parser.parse_args() + if args.cmake_dir: + work_dir = os.path.abspath(args.cmake_dir) + create_zipfile(args.limited_api) diff --git a/sources/shiboken2/libshiboken/embed/module_collector.py b/sources/shiboken2/libshiboken/embed/module_collector.py new file mode 100644 index 000000000..3eaa0be5d --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/module_collector.py @@ -0,0 +1,105 @@ +# This Python file uses the following encoding: utf-8 +# It has been edited by fix-complaints.py . + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of Qt for Python. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +module_collector.py + +Collect a number of modules listed on the command line. + +The purpose of this script is to generate the scripts needed for +a complete isolation of the signature extension. + +Usage: + +Run this script in one of the used python versions. +It will create an executable archive of the files on the command line. +""" + +import sys +import os +import argparse +import pickle +from textwrap import dedent + +def source_archive(module, modname): + fname = os.path.splitext(module.__file__)[0] + ".py" + with open(fname) as source: + text = source.read() + encoded = text.replace("'''", "(triple_single)") + # modname = module.__name__ + # Do not use: Some modules rename themselves! + version = ".".join(map(str, sys.version_info[:3])) + shortname = os.path.basename(fname) + preamble = dedent(r""" + # BEGIN SOURCE ARCHIVE Python {version} module {modname} + + sources = {{}} if "sources" not in globals() else sources + sources["{modname}"] = '''\ + {encoded}'''.replace("(triple_single)", "'''") + + # END SOURCE ARCHIVE Python {version} module {modname} + """).format(**locals()) + return preamble + +def read_all(modules): + collected = "" + for modname in modules: + mod = __import__(modname) + collected += source_archive(mod, modname) + return collected + +def license_header(): + license = os.path.join(os.path.dirname(__file__), "qt_python_license.txt") + with open(license) as f: + return f.read() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('modules', nargs="+") + args = parser.parse_args() + print("modules:", args.modules) + ret = license_header() + read_all(args.modules) + ma_mi = "_".join(map(str, sys.version_info[:2])) + outpath = os.path.join(os.path.dirname(__file__), "..", "..", "shibokenmodule", + "files.dir", "shibokensupport", "python_minilib_{ma_mi}.py".format(**locals())) + with open(outpath, "w") as f: + f.write(ret) diff --git a/sources/shiboken2/libshiboken/embed/qt_python_license.txt b/sources/shiboken2/libshiboken/embed/qt_python_license.txt new file mode 100644 index 000000000..b5f8c581a --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/qt_python_license.txt @@ -0,0 +1,87 @@ +# This Python file uses the following encoding: utf-8 +# It has been edited by fix-complaints.py . + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of Qt for Python. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +## +## PSF LICENSE AGREEMENT FOR PYTHON 3.7.0 +## +## 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and +## the Individual or Organization ("Licensee") accessing and otherwise using Python +## 3.7.0 software in source or binary form and its associated documentation. +## +## 2. Subject to the terms and conditions of this License Agreement, PSF hereby +## grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +## analyze, test, perform and/or display publicly, prepare derivative works, +## distribute, and otherwise use Python 3.7.0 alone or in any derivative +## version, provided, however, that PSF's License Agreement and PSF's notice of +## copyright, i.e., "Copyright © 2001-2018 Python Software Foundation; All Rights +## Reserved" are retained in Python 3.7.0 alone or in any derivative version +## prepared by Licensee. +## +## 3. In the event Licensee prepares a derivative work that is based on or +## incorporates Python 3.7.0 or any part thereof, and wants to make the +## derivative work available to others as provided herein, then Licensee hereby +## agrees to include in any such work a brief summary of the changes made to Python +## 3.7.0. +## +## 4. PSF is making Python 3.7.0 available to Licensee on an "AS IS" basis. +## PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF +## EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR +## WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE +## USE OF PYTHON 3.7.0 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. +## +## 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 3.7.0 +## FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF +## MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 3.7.0, OR ANY DERIVATIVE +## THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. +## +## 6. This License Agreement will automatically terminate upon a material breach of +## its terms and conditions. +## +## 7. Nothing in this License Agreement shall be deemed to create any relationship +## of agency, partnership, or joint venture between PSF and Licensee. This License +## Agreement does not grant permission to use PSF trademarks or trade name in a +## trademark sense to endorse or promote products or services of Licensee, or any +## third party. +## +## 8. By copying, installing or otherwise using Python 3.7.0, Licensee agrees +## to be bound by the terms and conditions of this License Agreement. +## diff --git a/sources/shiboken2/libshiboken/embed/signature_bootstrap.py b/sources/shiboken2/libshiboken/embed/signature_bootstrap.py new file mode 100644 index 000000000..6ce5ab95a --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/signature_bootstrap.py @@ -0,0 +1,178 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +signature_bootstrap.py +---------------------- + +This file was originally directly embedded into the C source. +After it grew more and more, I now prefer to have it as Python file. + +Meanwhile, there is also no more a stub loader necessary: +Because we meanwhile have embedding support, we could also load this file +directly from a .pyc file. + +This file replaces the hard to read Python stub in 'signature.cpp', and we +could distinguish better between bootstrap related functions and loader +functions. +It is embedded into 'signature.cpp' as "embed/signature_bootstrap.inc". +""" + +from __future__ import print_function, absolute_import + +recursion_trap = 0 + +# We avoid real imports in phase 1 that could fail (simply removed all). +# Python 2 is not able to import when the extension import is still active. +# Phase 1 simply defines the functions, which will be used in Phase 2. + +def bootstrap(): + import sys + import os + import tempfile + import traceback + from contextlib import contextmanager + + global recursion_trap + if recursion_trap: + # we are probably called from outside, already + print("Recursion occurred in Bootstrap. Did you start by hand? Then it's ok.") + print("But you should trigger start by 'type.__signature__', only!") + recursion_trap += 1 + + @contextmanager + def ensure_shibokensupport(support_path): + # Make sure that we always have the shibokensupport containing package first. + # Also remove any prior loaded module of this name, just in case. + sys.path.insert(0, support_path) + + sbks = "shibokensupport" + if sbks in sys.modules: + del sys.modules[sbks] + prefix = sbks + "." + for key in list(key for key in sys.modules if key.startswith(prefix)): + del sys.modules[key] + try: + import shibokensupport + yield + except Exception as e: + print("Problem importing shibokensupport:") + print(e) + traceback.print_exc() + print("sys.path:") + for p in sys.path: + print(" " + p) + sys.stdout.flush() + sys.exit(-1) + sys.path.remove(support_path) + + try: + import shiboken2 as root + except ImportError: + # uninstalled case without ctest, try only this one which has __init__: + import shibokenmodule as root + rp = os.path.realpath(os.path.dirname(root.__file__)) + # This can be the shiboken2 directory or the binary module, so search. + look_for = "files.dir" + while len(rp) > 3 and not os.path.exists(os.path.join(rp, look_for)): + rp = os.path.abspath(os.path.join(rp, "..")) + + # Here we decide if we work embedded or not. + embedding_var = "pyside_uses_embedding" + use_embedding = bool(getattr(sys, embedding_var, False)) + # We keep the zip file for inspection if the sys variable has been set. + keep_zipfile = hasattr(sys, embedding_var) + real_dir = os.path.join(rp, look_for) + + # We report in sys what we used. We could put more here as well. + if not os.path.exists(real_dir): + use_embedding = True + support_path = prepare_zipfile() if use_embedding else real_dir + setattr(sys, embedding_var, use_embedding) + + try: + with ensure_shibokensupport(support_path): + from shibokensupport.signature import loader + + except Exception as e: + print('Exception:', e) + traceback.print_exc(file=sys.stdout) + + finally: + if use_embedding and not keep_zipfile: + # clear the temp zipfile + try: + os.remove(support_path) + except OSError as e: + print(e) + print("Error deleting {support_path}, ignored".format(**locals())) + return loader + +# New functionality: Loading from a zip archive. +# There exists the zip importer, but as it is written, only real zip files are +# supported. Before I will start an own implementation, it is easiest to use +# a temporary zip file. + +def prepare_zipfile(): + """ + Write the zip file to a real file and return its name. + It will be implicitly opened as such when we add the name to sys.path . + """ + import base64 + import tempfile + import os + import zipfile + + # 'zipstring_sequence' comes from signature.cpp + zipbytes = base64.b64decode(''.join(zipstring_sequence)) + fd, fname = tempfile.mkstemp(prefix='embedded.', suffix='.zip') + os.write(fd, zipbytes) + os.close(fd) + # Let us test the zipfile if it really is one. + # Otherwise, zipimporter would simply ignore it without notice. + try: + z = zipfile.ZipFile(fname) + z.close() + except zipfile.BadZipFile as e: + print('Broken Zip File:', e) + traceback.print_exc(file=sys.stdout) + finally: + return fname + +# eof diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp index a99927c15..dea4c9b09 100644 --- a/sources/shiboken2/libshiboken/signature.cpp +++ b/sources/shiboken2/libshiboken/signature.cpp @@ -418,75 +418,101 @@ GetSignature_Cached(PyObject *props, const char *sig_kind, const char *modifier) return Py_INCREF(value), value; } -// const char *PySide_SignatureModule[] = { -// #include "embed/signature.inc" -// }; - -static const char PySide_PythonCode[] = - "from __future__ import print_function, absolute_import\n" R"~(if True: - - # This is becoming the 'signature_loader' module. - - import sys, os, traceback - # We avoid imports in phase 1 that could fail. "import shiboken" of the - # binary would even crash in FinishSignatureInitialization. - - def bootstrap(): - global __file__ - try: - import shiboken2 as root - except ImportError: - # uninstalled case without ctest, try only this one which has __init__: - import shibokenmodule as root - rp = os.path.realpath(os.path.dirname(root.__file__)) - # This can be the shiboken2 directory or the binary module, so search. - while len(rp) > 3 and not os.path.exists(os.path.join(rp, 'files.dir')): - rp = os.path.abspath(os.path.join(rp, '..')) - __file__ = os.path.join(rp, 'files.dir', 'shibokensupport', 'signature', 'loader.py') - try: - with open(__file__) as _f: - exec(compile(_f.read(), __file__, 'exec')) - except Exception as e: - print('Exception:', e) - traceback.print_exc(file=sys.stdout) - globals().update(locals()) - - )~"; +static const char *PySide_CompressedSignaturePackage[] = { +#include "embed/signature.inc" + }; + +static const unsigned char PySide_SignatureLoader[] = { +#include "embed/signature_bootstrap.inc" + }; static safe_globals_struc * init_phase_1(void) { - PyObject *d, *v; - safe_globals_struc *p = (safe_globals_struc *) - malloc(sizeof(safe_globals_struc)); - if (p == NULL) - goto error; - p->helper_module = PyImport_AddModule((char *) "signature_loader"); - if (p->helper_module == NULL) - goto error; - - // Initialize the module - d = PyModule_GetDict(p->helper_module); - if (PyDict_SetItemString(d, "__builtins__", PyEval_GetBuiltins()) < 0) - goto error; - v = PyRun_String(PySide_PythonCode, Py_file_input, d, d); - if (v == NULL) - goto error; - Py_DECREF(v); - - // build a dict for diverse mappings - p->map_dict = PyDict_New(); - if (p->map_dict == NULL) - goto error; - - // build a dict for the prepared arguments - p->arg_dict = PyDict_New(); - if (p->arg_dict == NULL - || PyObject_SetAttrString(p->helper_module, "pyside_arg_dict", p->arg_dict) < 0) - goto error; - return p; + { + safe_globals_struc *p = (safe_globals_struc *) + malloc(sizeof(safe_globals_struc)); + if (p == NULL) + goto error; + /* + * Initializing module signature_bootstrap. + * Since we now have an embedding script, we can do this without any + * Python strings in the C code. + */ +#ifdef Py_LIMITED_API + // We must work for multiple versions, so use source code. +#else + Shiboken::AutoDecRef marshal_str(Py_BuildValue("s", "marshal")); + if (marshal_str.isNull()) + goto error; + Shiboken::AutoDecRef marshal_module(PyImport_Import(marshal_str)); + if (marshal_module.isNull()) + goto error; + Shiboken::AutoDecRef loads(PyObject_GetAttrString(marshal_module, "loads")); + if (loads.isNull()) + goto error; +#endif + char *bytes_cast = reinterpret_cast<char *>( + const_cast<unsigned char *>(PySide_SignatureLoader)); + Shiboken::AutoDecRef bytes(PyBytes_FromStringAndSize(bytes_cast, + sizeof(PySide_SignatureLoader))); + if (bytes.isNull()) + goto error; +#ifdef Py_LIMITED_API + PyObject *builtins = PyEval_GetBuiltins(); + PyObject *compile = PyDict_GetItemString(builtins, "compile"); + if (compile == nullptr) + goto error; + Shiboken::AutoDecRef code_obj(PyObject_CallFunction(compile, "Oss", + bytes.object(), "(builtin)", "exec")); +#else + Shiboken::AutoDecRef code_obj(PyObject_CallFunctionObjArgs( + loads, bytes.object(), nullptr)); +#endif + if (code_obj.isNull()) + goto error; + p->helper_module = PyImport_ExecCodeModule(const_cast<char *> + ("signature_bootstrap"), code_obj); + if (p->helper_module == nullptr) + goto error; + // Initialize the module + PyObject *mdict = PyModule_GetDict(p->helper_module); + if (PyDict_SetItemString(mdict, "__builtins__", PyEval_GetBuiltins()) < 0) + goto error; + /* + * Unpack an embedded ZIP file with more signature modules. + * They will be loaded later with the zipimporter. + * Due to MSVC's limitation to 64k strings, we need to assemble pieces. + */ + const char **block_ptr = (const char **)PySide_CompressedSignaturePackage; + int npieces = 0; + PyObject *piece, *zipped_string_sequence = PyList_New(0); + for (; **block_ptr != 0; ++block_ptr) { + npieces++; + // we avoid the string/unicode dilemma by not using PyString_XXX: + piece = Py_BuildValue("s", *block_ptr); + if (piece == NULL || PyList_Append(zipped_string_sequence, piece) < 0) + goto error; + } + if (PyDict_SetItemString(mdict, "zipstring_sequence", zipped_string_sequence) < 0) + goto error; + Py_DECREF(zipped_string_sequence); + + // build a dict for diverse mappings + p->map_dict = PyDict_New(); + if (p->map_dict == NULL) + goto error; + + // build a dict for the prepared arguments + p->arg_dict = PyDict_New(); + if (p->arg_dict == NULL + || PyObject_SetAttrString(p->helper_module, "pyside_arg_dict", p->arg_dict) < 0) + goto error; + return p; + } error: + PyErr_Print(); PyErr_SetString(PyExc_SystemError, "could not initialize part 1"); return NULL; } @@ -494,38 +520,40 @@ error: static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) { - PyObject *bootstrap_func, *v = nullptr; - PyMethodDef *ml; - - // The single function to be called, but maybe more to come. - for (ml = methods; ml->ml_name != NULL; ml++) { - v = PyCFunction_NewEx(ml, nullptr, nullptr); - if (v == nullptr - || PyObject_SetAttrString(p->helper_module, ml->ml_name, v) != 0) + { + PyMethodDef *ml; + + // The single function to be called, but maybe more to come. + for (ml = methods; ml->ml_name != NULL; ml++) { + PyObject *v = PyCFunction_NewEx(ml, nullptr, nullptr); + if (v == nullptr + || PyObject_SetAttrString(p->helper_module, ml->ml_name, v) != 0) + goto error; + Py_DECREF(v); + } + PyObject *bootstrap_func = PyObject_GetAttrString(p->helper_module, "bootstrap"); + if (bootstrap_func == NULL) + goto error; + // The return value of the bootstrap function is the loader module. + PyObject *loader = PyObject_CallFunction(bootstrap_func, (char *)"()"); + if (loader == nullptr) goto error; - Py_DECREF(v); + // now the loader should be initialized + p->sigparse_func = PyObject_GetAttrString(loader, "pyside_type_init"); + if (p->sigparse_func == NULL) + goto error; + p->createsig_func = PyObject_GetAttrString(loader, "create_signature"); + if (p->createsig_func == NULL) + goto error; + p->seterror_argument_func = PyObject_GetAttrString(loader, "seterror_argument"); + if (p->seterror_argument_func == NULL) + goto error; + p->make_helptext_func = PyObject_GetAttrString(loader, "make_helptext"); + if (p->make_helptext_func == NULL) + goto error; + return 0; } - bootstrap_func = PyObject_GetAttrString(p->helper_module, "bootstrap"); - if (bootstrap_func == NULL - || PyObject_CallFunction(bootstrap_func, (char *)"()") == NULL) - goto error; - // now the loader should be initialized - p->sigparse_func = PyObject_GetAttrString(p->helper_module, "pyside_type_init"); - if (p->sigparse_func == NULL) - goto error; - p->createsig_func = PyObject_GetAttrString(p->helper_module, "create_signature"); - if (p->createsig_func == NULL) - goto error; - p->seterror_argument_func = PyObject_GetAttrString(p->helper_module, "seterror_argument"); - if (p->seterror_argument_func == NULL) - goto error; - p->make_helptext_func = PyObject_GetAttrString(p->helper_module, "make_helptext"); - if (p->make_helptext_func == NULL) - goto error; - return 0; - error: - Py_XDECREF(v); PyErr_Print(); PyErr_SetString(PyExc_SystemError, "could not initialize part 2"); return -1; diff --git a/sources/shiboken2/libshiboken/signature_doc.rst b/sources/shiboken2/libshiboken/signature_doc.rst index f51649b5b..9c42c5976 100644 --- a/sources/shiboken2/libshiboken/signature_doc.rst +++ b/sources/shiboken2/libshiboken/signature_doc.rst @@ -50,7 +50,7 @@ result of the ``__signature__`` attribute of the real ``PyCFunction`` object. There is one thing that really changes Python a bit: -* I added the ``__signature__`` attribute to every function. +* We added the ``__signature__`` attribute to every function. That is a little change to Python that does not harm, but it saves us tons of code, that was needed in the early versions of the module. @@ -59,9 +59,9 @@ The internal work is done in two steps: * All functions of a class get the *signature text* when the module is imported. This is only a very small overhead added to the startup time. It is a single - string for the whole class. + string for each whole class. * The actual signature object is created later, when the attribute is really - accessed. Signatures are cached and only created on first access. + requested. Signatures are cached and only created on first access. Example: @@ -76,10 +76,12 @@ Why this Code is Fast It costs a little time (maybe 4 seconds) to run througs every single signature object, since these are more than 15000 Python objects. But all the signature objects will be rarely accessed but in special applications. -The normal case are only a few accesses, and these work pretty fast. +The normal case are only a few accesses, and these are working pretty fast. The key to make this signature module fast is to avoid computation as much as -possible. When no signature objects are used, then no time is lost in initialization. +possible. When no signature objects are used, then almost no time is lost in +initialization. Only the above mentioned strings and some support modules are +additionally loaded on ``import PySide2``. When it comes to signature usage, then late initialization is used and cached. This technique is also known as *full laziness* in haskell. @@ -107,15 +109,27 @@ The C++ code involved with the signature module is completely in the file shiboken2/libshiboken/signature.cpp . All other functionality is implemented in the ``signature`` Python package. It has the following structure:: - pyside2/PySide2/support/signature/__init__.py - loader.py - parser.py - mapping.py - typing27.py - backport_inspect.py + shiboken2/files.dir/shibokensupport/ + backport_inspect.py + python_minilib_2_7.py + python_minilib_3_5.py + python_minilib_3_6.py + python_minilib_3_7.py -Really important are the **parser**, **mapping** and **loader** modules. The rest is -needed to create Python 2 compatibility. + signature/ + loader.py + parser.py + mapping.py + errorhandler.py + layout.py + + lib/ + enum_sig.py + + +Really important are the **parser**, **mapping**, **errorhandler**, **enum_sig**, +**layout** and **loader** modules. The rest is needed to create Python 2 compatibility +or be compatible with embedding and installers. loader.py @@ -143,6 +157,34 @@ needs. A lot of mappings are resolved by rather complex expressions in ``parser. but a few hundred cases are better to spell explicitly, here. +errorhandler.py +~~~~~~~~~~~~~~~ + +Since ``Qt For Python 5.12``, we no longer use the builtin type error messages from C++. +Instead, we get much better results with the signature module. At the same time, +this enforced supporting shiboken as well, and the signature module was no longer +optional. + + +enum_sig.py +~~~~~~~~~~~ + +The diverse applications of the signature module all needed to iterate over modules, +classes and functions. In order to centralize this enumeration, the process has +been factored out as a context manager. The user has only to supply functions +that do the actual formatting. + +See for example the .pyi generator ``pyside2/PySide2/support/generate_pyi.py``. + + +layout.py +~~~~~~~~~ + +As more applications used the signature module, different formatting of signatures +was needed. To support that, we created the function ``create_signature``, which +has a parameter to choose from some prefefined layouts. + + *typing27.py* ~~~~~~~~~~~~~ @@ -276,19 +318,21 @@ This serves as an extra challenge that has a very positive effect on the completeness and correctness of signatures. -Future Extension ----------------- +Current Extensions +------------------ Before the signature module was written, there already existed the concept of -signatures, but in a more C++ - centric way. From that time, there still exist +signatures, but in a more C++ - centric way. From that time, there existed the error messages, which are created when a function gets wrong argument types. -These error messages should be replaced by text generated on demand by +These error messages were replaced by text generated on demand by the signature module, in order to be more consistent and correct. +This was implemented in ``Qt For Python 5.12.0``. -Additionally, the ``__doc__`` attribute of PySide methods is not set, yet. -It would be easy to get a nice ``help()`` feature by creating signatures +Additionally, the ``__doc__`` attribute of PySide methods was not set. +It was easy to get a nice ``help()`` feature by creating signatures as default content for docstrings. +This was implemented in ``Qt For Python 5.12.1``. Literature diff --git a/sources/shiboken2/shibokenmodule/CMakeLists.txt b/sources/shiboken2/shibokenmodule/CMakeLists.txt index 612055b31..61c38c5d7 100644 --- a/sources/shiboken2/shibokenmodule/CMakeLists.txt +++ b/sources/shiboken2/shibokenmodule/CMakeLists.txt @@ -71,10 +71,10 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/ "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/lib/enum_sig.py" COPYONLY) if (PYTHON_VERSION_MAJOR EQUAL 3) else() - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/backport_inspect.py" - "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/backport_inspect.py" COPYONLY) - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/typing27.py" - "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/typing27.py" COPYONLY) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/backport_inspect.py" + "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/backport_inspect.py" COPYONLY) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/typing27.py" + "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/typing27.py" COPYONLY) endif() install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/files.dir" DESTINATION "${PYTHON_SITE_PACKAGES}/shiboken2") diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py index fbd82cc53..c690493b6 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py @@ -111,10 +111,9 @@ CO_NOFREE = 0x0040 # This function was changed: 'builtins' and 'qualname' don't exist. # We use '__builtin__' and '__name__' instead. -# It is further changed because we use a local copy of typing def formatannotation(annotation, base_module=None): - if getattr(annotation, '__module__', None) == 'support.signature.typing27': - return repr(annotation).replace('support.signature.typing27', 'typing') + if getattr(annotation, '__module__', None) == 'typing': + return repr(annotation).replace('typing.', '') if isinstance(annotation, type): if annotation.__module__ in ('__builtin__', base_module): return annotation.__name__ @@ -378,7 +377,7 @@ class Parameter(object): # Add annotation and default value if self._annotation is not _empty: - formatted = '{}:{}'.format(formatted, + formatted = '{}: {}'.format(formatted, formatannotation(self._annotation)) if self._default is not _empty: @@ -892,65 +891,3 @@ class Signature(object): def signature(obj, follow_wrapped=True): """Get a signature object for the passed callable.""" return Signature.from_callable(obj, follow_wrapped=follow_wrapped) - - -def _main(): - """ Logic for inspecting an object given at command line """ - import argparse - import importlib - - parser = argparse.ArgumentParser() - parser.add_argument( - 'object', - help="The object to be analysed. " - "It supports the 'module:qualname' syntax") - parser.add_argument( - '-d', '--details', action='store_true', - help='Display info about the module rather than its source code') - - args = parser.parse_args() - - target = args.object - mod_name, has_attrs, attrs = target.partition(":") - try: - obj = module = importlib.import_module(mod_name) - except Exception as exc: - msg = "Failed to import {} ({}: {})".format(mod_name, - type(exc).__name__, - exc) - print(msg, file=sys.stderr) - exit(2) - - if has_attrs: - parts = attrs.split(".") - obj = module - for part in parts: - obj = getattr(obj, part) - - if module.__name__ in sys.builtin_module_names: - print("Can't get info for builtin modules.", file=sys.stderr) - exit(1) - - if args.details: - print('Target: {}'.format(target)) - print('Origin: {}'.format(getsourcefile(module))) - print('Cached: {}'.format(module.__cached__)) - if obj is module: - print('Loader: {}'.format(repr(module.__loader__))) - if hasattr(module, '__path__'): - print('Submodule search path: {}'.format(module.__path__)) - else: - try: - __, lineno = findsource(obj) - except Exception: - pass - else: - print('Line: {}'.format(lineno)) - - print('\n') - else: - print(getsource(obj)) - - -if __name__ == "__main__": - _main() diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py index cdd84f9be..3818ad602 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py @@ -1,3 +1,6 @@ +# This Python file uses the following encoding: utf-8 +# It has been edited by fix-complaints.py . + ############################################################################# ## ## Copyright (C) 2019 The Qt Company Ltd. @@ -42,14 +45,15 @@ from __future__ import print_function, absolute_import """ fix-complaints.py -This module fixes the buildbot messages of external python modules. +This module fixes the buildbot messages of external python files. Run it once after copying a new version. It is idem-potent, unless you are changing messages (what I did, of course :-) . """ import os +import glob -patched_modules = "backport_inspect typing27" +patched_file_patterns = "backport_inspect.py typing27.py python_minilib_*.py" offending_words = { "behavio""ur": "behavior", @@ -79,9 +83,12 @@ def patch_file(fname): f.write("".join(lines)) def doit(): - dir = os.path.dirname(__file__) - for name in patched_modules.split(): - fname = os.path.join(dir, name + ".py") + dirname = os.path.dirname(__file__) + patched_files = [] + for name in patched_file_patterns.split(): + pattern = os.path.join(dirname, name) + patched_files += glob.glob(pattern) + for fname in patched_files: print("Working on", fname) patch_file(fname) diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py index 9447e9c25..ee541d0ea 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py @@ -39,6 +39,4 @@ from __future__ import print_function, absolute_import -# from shibokensupport.signature import get_signature, inspect, typing -# This gives a problem with Python 2. We do it in the loader, instead. -__all__ = "get_signature inspect typing layout mapping lib".split() +__all__ = "get_signature layout mapping lib".split() diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py index 1443d7927..cb148830f 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py @@ -56,7 +56,8 @@ enough to produce a useful ValueError. This matter will be improved in a later version. """ -from shibokensupport.signature import get_signature, inspect +from shibokensupport.signature import inspect +from shibokensupport.signature import get_signature from shibokensupport.signature.mapping import update_mapping, namespace from textwrap import dedent diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py index 90affbcfd..e6f6dc379 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py @@ -50,7 +50,8 @@ by producing a lot of clarity. """ import sys -from shibokensupport.signature import get_signature, inspect +from shibokensupport.signature import inspect +from shibokensupport.signature import get_signature class ExactEnumerator(object): diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py index 3a8d614db..a564aedb9 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py @@ -1,3 +1,6 @@ +# This Python file uses the following encoding: utf-8 +# It has been edited by fix-complaints.py . + ############################################################################# ## ## Copyright (C) 2019 The Qt Company Ltd. @@ -42,33 +45,26 @@ from __future__ import print_function, absolute_import """ loader.py -The loader has to lazy-load the signature module and also provides a few -Python modules to support Python 2.7 . +The loader has to load the signature module completely at startup, +to make sure that the functions are available when needed. +This is meanwhile necessary to make the '__doc__' attribute work correctly. + +It does not mean that everything is initialized in advance. Only the modules +are loaded completely after 'import PySide2'. + +This version uses both a normal directory, but has also an embedded ZIP file +as a fallback solution. The ZIP file is generated by 'embedding_generator.py' +and embedded into 'signature.cpp' as "embed/signature.inc". -This version uses both a normal directory, but has also an embedded zip file -as a fallback solution. +Meanwhile, the ZIP file grew so much, that MSVC had problems +with it's 64k string limit, so we had to break the string up. +See 'zipped_string_sequence' in signature.cpp. """ import sys import os import traceback import types -from contextlib import contextmanager - -""" -A note on the import problem (solved): - -During the tests, the shiboken build structure has the layout - - shiboken2/shibokenmodule/shiboken2.abi3.so - -and the name "shiboken2" in sys.modules points directly to the binary -file, hiding the outer shiboken2 module. - -To fix that, we temporarily remove the binary from sys.path, -do the needed imports and then restore the binary. -This action was put into a context manager for readability. -""" # On Python 2, we only have ImportError, which is way too coarse. # When problems occour, please use Python 3, because it has the finer @@ -79,31 +75,6 @@ try: except NameError: ModuleNotFoundError = ImportError -@contextmanager -def ensure_import_shibokensupport(): - # Make sure that we always have the shibokensupport containing package first. - # Also remove any prior loaded module of this name, just in case. - sbk_support_dir = os.path.abspath(os.path.join(__file__, "..", "..", "..")) - assert os.path.basename(sbk_support_dir) == "files.dir" - sys.path.insert(0, sbk_support_dir) - - sbk = "shibokensupport" - if sbk in sys.modules: - del sys.modules[sbk] - for key in list(key for key in sys.modules if key.startswith(sbk + ".")): - del sys.modules[key] - try: - import shibokensupport - yield - except Exception as e: - print("Problem importing shibokensupport:") - print(e) - traceback.print_exc() - sys.stdout.flush() - sys.exit(-1) - sys.path.remove(sbk_support_dir) - - # patching inspect's formatting to keep the word "typing": def formatannotation(annotation, base_module=None): # if getattr(annotation, '__module__', None) == 'typing': @@ -143,53 +114,102 @@ def seterror_argument(args, func_name): def make_helptext(func): return errorhandler.make_helptext(func) -with ensure_import_shibokensupport(): - import signature_loader - import shibokensupport.signature - shibokensupport.signature.get_signature = signature_loader.get_signature - del signature_loader # protect this dir, too? +import signature_bootstrap +from shibokensupport import signature +signature.get_signature = signature_bootstrap.get_signature +del signature_bootstrap + - if sys.version_info >= (3,): - import typing - import inspect - inspect.formatannotation = formatannotation +def _get_modname(mod): + return mod.__spec__.name if getattr(mod, "__spec__", None) else mod.__name__ + +def _set_modname(mod, name): + if getattr(mod, "__spec__", None): + mod.__spec__.name = name else: - import inspect - namespace = inspect.__dict__ - from shibokensupport.signature import typing27 as typing - typing.__name__ = "typing" - # Fix the module names in typing if possible. This is important since - # the typing names should be I/O compatible, so that typing.Dict - # shows itself as "typing.Dict". - for name, obj in typing.__dict__.items(): - if hasattr(obj, "__module__"): - try: - obj.__module__ = "typing" - except (TypeError, AttributeError): - pass - from shibokensupport.signature import backport_inspect as inspect - _doc = inspect.__doc__ - inspect.__dict__.update(namespace) - inspect.__doc__ += _doc - # force inspect to find all attributes. See "heuristic" in pydoc.py! - inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_")) - typing.TypeVar.__repr__ = _typevar__repr__ - - def put_into_package(module, package): - # take the last component of the module name - name = module.__name__.rsplit(".", 1)[-1] - # allow access as {package}.typing + mod.__name__ = name + + +def put_into_package(package, module, override=None): + # take the last component of the module name + name = (override if override else _get_modname(module)).rsplit(".", 1)[-1] + # allow access as {package}.typing + if package: setattr(package, name, module) - # put into sys.modules as a package to allow all import options - fullname = "{}.{}".format(package.__name__, name) - sys.modules[fullname] = module - - put_into_package(typing, shibokensupport.signature) - put_into_package(inspect, shibokensupport.signature) - from shibokensupport.signature import mapping - from shibokensupport.signature import errorhandler - from shibokensupport.signature import layout - from shibokensupport.signature.lib import enum_sig - from shibokensupport.signature.parser import pyside_type_init + # put into sys.modules as a package to allow all import options + fullname = "{}.{}".format(_get_modname(package), name) if package else name + _set_modname(module, fullname) + # publish new dotted name in sys.modules + sys.modules[fullname] = module + + +# Debug: used to inspect what each step loads +def list_modules(message): + ext_modules = {key:value for (key, value) in sys.modules.items() + if hasattr(value, "__file__")} + print("SYS.MODULES", message, len(sys.modules), len(ext_modules)) + for (name, module) in sorted(ext_modules.items()): + print(" {:23}".format(name), repr(module)[:70]) + + +if sys.version_info >= (3,): + import typing + import inspect + inspect.formatannotation = formatannotation +else: + from shibokensupport import typing27 as typing + import inspect + namespace = inspect.__dict__ + from shibokensupport import backport_inspect as inspect + _doc = inspect.__doc__ + inspect.__dict__.update(namespace) + inspect.__doc__ += _doc + # force inspect to find all attributes. See "heuristic" in pydoc.py! + inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_")) +typing.TypeVar.__repr__ = _typevar__repr__ + +# Fix the module names in typing if possible. This is important since +# the typing names should be I/O compatible, so that typing.Dict +# shows itself as "typing.Dict". +for name, obj in typing.__dict__.items(): + if hasattr(obj, "__module__"): + try: + obj.__module__ = "typing" + except (TypeError, AttributeError): + pass + +import shibokensupport +put_into_package(shibokensupport.signature, typing, "typing") +put_into_package(shibokensupport.signature, inspect, "inspect") + + +def move_into_pyside_package(): + import PySide2 + try: + import PySide2.support + except ModuleNotFoundError: + PySide2.support = types.ModuleType("PySide2.support") + put_into_package(PySide2.support, signature) + put_into_package(PySide2.support.signature, mapping) + put_into_package(PySide2.support.signature, errorhandler) + put_into_package(PySide2.support.signature, layout) + put_into_package(PySide2.support.signature, lib) + put_into_package(PySide2.support.signature, parser) + put_into_package(PySide2.support.signature.lib, enum_sig) + + put_into_package(PySide2.support.signature, typing) + put_into_package(PySide2.support.signature, inspect) + +from shibokensupport.signature import mapping +from shibokensupport.signature import errorhandler +from shibokensupport.signature import layout +from shibokensupport.signature import lib +from shibokensupport.signature import parser +from shibokensupport.signature.lib import enum_sig +from shibokensupport.signature.parser import pyside_type_init + +if "PySide2" in sys.modules: + # We publish everything under "PySide2.support.signature", again. + move_into_pyside_package() # end of file diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py index 470dd676b..9a8ee4c4e 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py @@ -52,7 +52,6 @@ in sys.modules. This minimizes the loading overhead. import sys import struct import os -import pkgutil from shibokensupport.signature import typing from shibokensupport.signature.typing import TypeVar, Generic @@ -168,9 +167,7 @@ class _NotCalled(str): real object is needed, the wrapper can simply be called. """ def __repr__(self): - suppress = "support.signature.typing27." - text = self[len(suppress):] if self.startswith(suppress) else self - return "{}({})".format(type(self).__name__, text) + return "{}({})".format(type(self).__name__, self) def __call__(self): from shibokensupport.signature.mapping import __dict__ as namespace @@ -346,6 +343,7 @@ def init_other(): def init_smart(): type_map.update({ "smart.SharedPtr": Missing("smart.SharedPtr"), # bad object "SharedPtr<Obj >" + "smart.Smart.Integer2": int, }) return locals() diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json index 491ae8054..fbe4c51ab 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json @@ -3,7 +3,7 @@ "Name": "Python", "QDocModule": "QtForPython", "QtUsage": "Used for Qt for Python in the signature extension.", - "Description": "Qt for Python is an add-on for Python. The signature packages of PySide uses certain copied and adapted source files (backport_inspect.py, typing27.py). See the folder sources/pyside2/PySide2/support/signature .", + "Description": "Qt for Python is an add-on for Python. The signature packages of PySide uses certain copied and adapted source files (backport_inspect.py, typing27.py). See the folder sources/shiboken2/files.dir/shibokensupport/signature .", "Homepage": "http://www.python.org/", "Version": "3.7.0", "LicenseId": "Python-2.0", diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py index dba8f8c77..dba8f8c77 100644 --- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py +++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py |