diff options
Diffstat (limited to 'sources/shiboken6/libshiboken/embed')
4 files changed, 181 insertions, 250 deletions
diff --git a/sources/shiboken6/libshiboken/embed/embedding_generator.py b/sources/shiboken6/libshiboken/embed/embedding_generator.py index f9ec60cbc..51c46ce54 100644 --- a/sources/shiboken6/libshiboken/embed/embedding_generator.py +++ b/sources/shiboken6/libshiboken/embed/embedding_generator.py @@ -1,41 +1,6 @@ -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of PySide6. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +from __future__ import annotations """ embedding_generator.py @@ -62,7 +27,7 @@ import marshal import traceback from pathlib import Path -# work_dir is set to the source for testing, onl. +# work_dir is set to the source for testing, only. # It can be overridden in the command line. work_dir = Path(__file__).parent.resolve() embed_dir = work_dir @@ -81,7 +46,7 @@ def runpy(cmd, **kw): subprocess.call([sys.executable, '-E'] + cmd.split(), **kw) -def create_zipfile(limited_api): +def create_zipfile(use_pyc, quiet): """ Collect all Python files, compile them, create a zip file and make a chunked base64 encoded file from it. @@ -110,7 +75,7 @@ def create_zipfile(limited_api): if embed_dir != work_dir: utils.copyfile(embed_dir / "signature_bootstrap.py", work_dir) - if limited_api: + if not use_pyc: pass # We cannot compile, unless we have folders per Python version else: files = ' '.join(fn for fn in os.listdir('.')) @@ -124,11 +89,26 @@ def create_zipfile(limited_api): with open(inc_name, "w") as inc: _embed_file(tmp, inc) tmp.close() + # also generate a simple embeddable .pyc file for signature_bootstrap.pyc - boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc" + boot_name = "signature_bootstrap.py" if not use_pyc else "signature_bootstrap.pyc" with open(boot_name, "rb") as ldr, open("signature_bootstrap_inc.h", "w") as inc: - _embed_bytefile(ldr, inc, limited_api) + _embed_bytefile(ldr, inc, not use_pyc) os.chdir(cur_dir) + if quiet: + return + + # have a look at our populated folder unless quiet option + def tree(directory): + print(f'+ {directory}') + for path in sorted(directory.rglob('*')): + depth = len(path.relative_to(directory).parts) + spacer = ' ' * depth + print(f'{spacer}+ {path.name}') + + print("++++ Current contents of") + tree(work_dir) + print("++++") def _embed_file(fin, fout): @@ -143,22 +123,23 @@ def _embed_file(fin, fout): limit = 50 text = fin.readlines() print(textwrap.dedent(""" - /* - * This is a ZIP archive of all Python files in the directory - * "shiboken6/shibokenmodule/files.dir/shibokensupport/signature" - * There is also a toplevel file "signature_bootstrap.py[c]" that will be - * directly executed from C++ as a bootstrap loader. - */ + // This is a ZIP archive of all Python files in the directory + // "shiboken6/shibokenmodule/files.dir/shibokensupport/signature" + // There is also a toplevel file "signature_bootstrap.py[c]" that will be + // directly executed from C++ as a bootstrap loader. """).strip(), file=fout) block, blocks = 0, len(text) // limit + 1 for idx, line in enumerate(text): if idx % limit == 0: + if block: + fout.write(')"\n') comma = "," if block else "" block += 1 - print(file=fout) - print(f'/* Block {block} of {blocks} */{comma}', file=fout) - print(f'\"{line.strip()}\"', file=fout) - print(f'/* Sentinel */, \"\"', file=fout) + fout.write(f'\n{comma} // Block {block} of {blocks}\nR"(') + else: + fout.write('\n') + fout.write(line.strip()) + fout.write(')"\n\n/* Sentinel */, ""\n') def _embed_bytefile(fin, fout, is_text): @@ -232,8 +213,9 @@ def str2bool(v): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--cmake-dir', nargs="?") - parser.add_argument('--limited-api', type=str2bool) + parser.add_argument('--use-pyc', type=str2bool) + parser.add_argument('--quiet', action='store_true') args = parser.parse_args() if args.cmake_dir: work_dir = Path(args.cmake_dir).resolve() - create_zipfile(args.limited_api) + create_zipfile(args.use_pyc, args.quiet) diff --git a/sources/shiboken6/libshiboken/embed/module_collector.py b/sources/shiboken6/libshiboken/embed/module_collector.py index 42ce2667c..8f7be6437 100644 --- a/sources/shiboken6/libshiboken/embed/module_collector.py +++ b/sources/shiboken6/libshiboken/embed/module_collector.py @@ -1,44 +1,6 @@ -# This Python file uses the following encoding: utf-8 -# It has been edited by fix-complaints.py . - -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +from __future__ import annotations """ module_collector.py diff --git a/sources/shiboken6/libshiboken/embed/qt_python_license.txt b/sources/shiboken6/libshiboken/embed/qt_python_license.txt index b5f8c581a..e5fdfdf4d 100644 --- a/sources/shiboken6/libshiboken/embed/qt_python_license.txt +++ b/sources/shiboken6/libshiboken/embed/qt_python_license.txt @@ -1,44 +1,5 @@ -# This Python file uses the following encoding: utf-8 -# It has been edited by fix-complaints.py . - -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only ## ## PSF LICENSE AGREEMENT FOR PYTHON 3.7.0 diff --git a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py index 4b48c0076..b0ba77107 100644 --- a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py +++ b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py @@ -1,41 +1,6 @@ -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of PySide6. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +from __future__ import annotations """ signature_bootstrap.py @@ -52,21 +17,27 @@ This file replaces the hard to read Python stub in 'signature.cpp', and we could distinguish better between bootstrap related functions and loader functions. It is embedded into 'signature.cpp' as "embed/signature_bootstrap.inc". + +# PYSIDE-1436: Python 3.10 had a problem with EmbeddableZipImporter because the +imports were in the functions. Moved them outside into the globals. """ recursion_trap = 0 -# We avoid real imports in phase 1 that could fail (simply removed all). -# Python 2 is not able to import when the extension import is still active. -# Phase 1 simply defines the functions, which will be used in Phase 2. +import base64 +import importlib +import io +import os +import sys +import traceback +import zipfile + +from contextlib import contextmanager +from importlib.machinery import ModuleSpec +from pathlib import Path + def bootstrap(): - import sys - import os - import tempfile - import traceback - from contextlib import contextmanager - from pathlib import Path global recursion_trap if recursion_trap: @@ -76,10 +47,11 @@ def bootstrap(): recursion_trap += 1 @contextmanager - def ensure_shibokensupport(support_path): + def ensure_shibokensupport(target, support_path): # Make sure that we always have the shibokensupport containing package first. # Also remove any prior loaded module of this name, just in case. - sys.path.insert(0, os.fspath(support_path)) + # PYSIDE-1621: support_path can also be a finder instance. + target.insert(0, support_path) sbks = "shibokensupport" if sbks in sys.modules: @@ -91,89 +63,143 @@ def bootstrap(): import shibokensupport yield except Exception as e: - print("Problem importing shibokensupport:") - print(e) + f = sys.stderr + print("Problem importing shibokensupport:", file=f) + print(f"{e.__class__.__name__}: {e}", file=f) traceback.print_exc() - print("sys.path:") + print("sys.path:", file=f) for p in sys.path: - print(" " + p) - sys.stdout.flush() + print(" " + p, file=f) + f.flush() sys.exit(-1) - sys.path.remove(os.fspath(support_path)) - + target.remove(support_path) + + # Here we decide if we re-incarnate the embedded files or use embedding. + incarnated = find_incarnated_files() + if incarnated: + target, support_path = sys.path, os.fspath(incarnated) + else: + target, support_path = prepare_zipfile() + with ensure_shibokensupport(target, support_path): + from shibokensupport.signature import loader + return loader + +# Newer functionality: +# This function checks if the support directory exist and returns it. +# If does not exist, we try to create it and return it. +# Otherwise, we return None. + +def find_incarnated_files(): import shiboken6 as root - path = Path(root.__file__) - rp = path.parent.resolve() - # This can be the shiboken6 directory or the binary module, so search. - look_for = Path("files.dir") / "shibokensupport" / "signature" / "loader.py" - while not (rp / look_for).exists(): - dir = rp.parent - if dir == rp: # Hit root, '/', 'C:\', '\\server\share' - break - rp = dir - - # Here we decide if we work embedded or not. - embedding_var = "pyside_uses_embedding" - use_embedding = bool(getattr(sys, embedding_var, False)) - # We keep the zip file for inspection if the sys variable has been set. - keep_zipfile = hasattr(sys, embedding_var) - loader_path = rp / look_for - files_dir = loader_path.parents[2] - assert files_dir.name == "files.dir" - - # We report in sys what we used. We could put more here as well. - if not loader_path.exists(): - use_embedding = True - support_path = Path(prepare_zipfile()) if use_embedding else files_dir - setattr(sys, embedding_var, use_embedding) - + files_dir = Path(root.__file__).resolve().parent / "files.dir" + handle_embedding_switch(files_dir) + if files_dir.exists(): + sys.path.insert(0, os.fspath(files_dir)) + # Note: To avoid recursion problems, we need to preload the loader. + # But that has the side-effect that we need to delay the feature + # initialization until all function pointers are set. + # See `post_init_func` in signature_globals.cpp . + import shibokensupport.signature.loader + del sys.path[0] + return files_dir + return None + + +def handle_embedding_switch(files_dir): + """ + This handles the optional environment variable `SBK_EMBED` + if not set : do nothing + if set to 0, false, no : de-virtualize the Python files + if set to 1, true, yes : virtualize again (delete "files.dir") + """ + env_name = "SBK_EMBED" + env_var = os.environ.get(env_name) + if not env_var: + return + if env_var.lower() in ("1", "t", "true", "y", "yes"): + import shutil + shutil.rmtree(files_dir, ignore_errors=True) + elif env_var.lower() in ("0", "f", "false", "n", "no"): + reincarnate_files(files_dir) + + +def reincarnate_files(files_dir): + target, zip = prepare_zipfile() + names = (_ for _ in zip.zfile.namelist() if _.endswith(".py")) try: - with ensure_shibokensupport(support_path): - from shibokensupport.signature import loader - + # First check mkdir to get an error when we cannot write. + files_dir.mkdir(exist_ok=True) + except os.error as e: + print(f"SBK_EMBED=False: Warning: Cannot write into {files_dir}") + return None + try: + # Then check for a real error when unpacking the zip file. + zip.zfile.extractall(path=files_dir, members=names) + return files_dir except Exception as e: - print('Exception:', e) - traceback.print_exc(file=sys.stdout) - - finally: - if use_embedding and not keep_zipfile: - # clear the temp zipfile - try: - os.remove(support_path) - except OSError as e: - print(e) - print(f"Error deleting {support_path}, ignored") - return loader + print(f"{e.__class__.__name__}: {e}", file=sys.stderr) + traceback.print_exc() + raise # New functionality: Loading from a zip archive. # There exists the zip importer, but as it is written, only real zip files are # supported. Before I will start an own implementation, it is easiest to use # a temporary zip file. +# PYSIDE-1621: make zip file access totally virtual def prepare_zipfile(): """ + Old approach: + Write the zip file to a real file and return its name. It will be implicitly opened as such when we add the name to sys.path . + + New approach (Python 3, only): + + Use EmbeddableZipImporter and pass the zipfile structure directly. + The sys.path way does not work, instead we need to use sys.meta_path . + See https://docs.python.org/3/library/sys.html#sys.meta_path """ - import base64 - import tempfile - import os - import zipfile # 'zipstring_sequence' comes from signature.cpp zipbytes = base64.b64decode(''.join(zipstring_sequence)) - fd, fname = tempfile.mkstemp(prefix='embedded.', suffix='.zip') - os.write(fd, zipbytes) - os.close(fd) - # Let us test the zipfile if it really is one. - # Otherwise, zipimporter would simply ignore it without notice. - try: - z = zipfile.ZipFile(fname) - z.close() - except zipfile.BadZipFile as e: - print('Broken Zip File:', e) - traceback.print_exc(file=sys.stdout) - finally: - return fname + vzip = zipfile.ZipFile(io.BytesIO(zipbytes)) + return sys.meta_path, EmbeddableZipImporter(vzip) + + +class EmbeddableZipImporter(object): + + def __init__(self, zip_file): + def p2m(filename): + if filename.endswith("/__init__.py"): + return filename[:-12].replace("/", ".") + if filename.endswith(".py"): + return filename[:-3].replace("/", ".") + return None + + self.zfile = zip_file + self._mod2path = {p2m(_.filename) : _.filename for _ in zip_file.filelist} + + def find_spec(self, fullname, path, target=None): + path = self._mod2path.get(fullname) + return ModuleSpec(fullname, self) if path else None + + def create_module(self, spec): + return None + + def exec_module(self, module): + fullname = module.__spec__.name + filename = self._mod2path[fullname] + with self.zfile.open(filename, "r") as f: # "rb" not for zipfile + codeob = compile(f.read(), filename, "exec") + exec(codeob, module.__dict__) + module.__file__ = filename + module.__loader__ = self + if filename.endswith("/__init__.py"): + module.__path__ = [] + module.__package__ = fullname + else: + module.__package__ = fullname.rpartition('.')[0] + sys.modules[fullname] = module # eof |