diff options
Diffstat (limited to 'sources/shiboken2/libshiboken/embed')
4 files changed, 611 insertions, 0 deletions
diff --git a/sources/shiboken2/libshiboken/embed/embedding_generator.py b/sources/shiboken2/libshiboken/embed/embedding_generator.py new file mode 100644 index 000000000..3ee96a1a5 --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/embedding_generator.py @@ -0,0 +1,241 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +embedding_generator.py + +This file takes the content of the two supported directories and inserts +it into a zip file. The zip file is then converted into a C++ source +file that can easily be unpacked again with Python (see signature.cpp, +constant 'PySide_PythonCode'). + +Note that this _is_ a zipfile, but since it is embedded into the shiboken +binary, we cannot use the zipimport module from Python. +But a similar solution is possible that allows for normal imports. + +See signature_bootstrap.py for details. +""" + +from __future__ import print_function, absolute_import + +import sys +import os +import subprocess +import textwrap +import tempfile +import argparse +import marshal +import traceback + +# work_dir is set to the source for testing, onl. +# It can be overridden in the command line. +work_dir = os.path.abspath(os.path.dirname(__file__)) +embed_dir = work_dir +cur_dir = os.getcwd() +source_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..")) +assert os.path.basename(source_dir) == "sources" +build_script_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..", "..")) +assert os.path.exists(os.path.join(build_script_dir, "build_scripts")) + +sys.path.insert(0, build_script_dir) + +from build_scripts import utils + + +def runpy(cmd, **kw): + subprocess.call([sys.executable, '-E'] + cmd.split(), **kw) + + +def create_zipfile(limited_api): + """ + Collect all Python files, compile them, create a zip file + and make a chunked base64 encoded file from it. + """ + zip_name = "signature.zip" + inc_name = "signature.inc" + flag = '-b' if sys.version_info >= (3,) else '' + os.chdir(work_dir) + + # Limited API: Remove all left-over py[co] files first, in case we use '--reuse-build'. + # Note that we could improve that with the PyZipfile function to use .pyc files + # in different folders, but that makes only sense when COIN allows us to have + # multiple Python versions in parallel. + from os.path import join, getsize + for root, dirs, files in os.walk(work_dir): + for name in files: + fpath = os.path.join(root, name) + if name.endswith(".pyc") or name.endswith(".pyo"): + os.remove(fpath) + + # We copy every Python file into this dir, but only for the right version. + # For testing in the source dir, we need to filter. + if sys.version_info[0] == 3: + ignore = "backport_inspect.py typing27.py".split() + else: + ignore = "".split() + utils.copydir(os.path.join(source_dir, "shiboken2", "shibokenmodule", "files.dir", "shibokensupport"), + os.path.join(work_dir, "shibokensupport"), + ignore=ignore, file_filter_function=lambda name, n2: name.endswith(".py")) + if embed_dir != work_dir: + utils.copyfile(os.path.join(embed_dir, "signature_bootstrap.py"), work_dir) + + if limited_api: + pass # We cannot compile, unless we have folders per Python version + else: + files = ' '.join(fn for fn in os.listdir('.')) + runpy('-m compileall -q {flag} {files}'.format(**locals())) + files = ' '.join(fn for fn in os.listdir('.') if not fn == zip_name) + runpy('-m zipfile -c {zip_name} {files}'.format(**locals())) + tmp = tempfile.TemporaryFile(mode="w+") + runpy('-m base64 {zip_name}'.format(**locals()), stdout=tmp) + # now generate the include file + tmp.seek(0) + with open(inc_name, "w") as inc: + _embed_file(tmp, inc) + # also generate a simple embeddable .pyc file for signature_bootstrap.pyc + boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc" + with open(boot_name, "rb") as ldr, open("signature_bootstrap.inc", "w") as inc: + _embed_bytefile(ldr, inc, limited_api) + os.chdir(cur_dir) + + +def _embed_file(fin, fout): + """ + Format a text file for embedding in a C++ source file. + """ + # MSVC has a 64k string limitation. In C, it would be easy to create an + # array of 64 byte strings and use them as one big array. In C++ this does + # not work, since C++ insists in having the terminating nullbyte. + # Therefore, we split the string after an arbitrary number of lines + # (chunked file). + limit = 50 + text = fin.readlines() + print(textwrap.dedent(""" + /* + * This is a ZIP archive of all Python files in the directory + * "shiboken2/shibokenmodule/files.dir/shibokensupport/signature" + * There is also a toplevel file "signature_bootstrap.py[c]" that will be + * directly executed from C++ as a bootstrap loader. + */ + """).strip(), file=fout) + block, blocks = 0, len(text) // limit + 1 + for idx, line in enumerate(text): + if idx % limit == 0: + comma = "," if block else "" + block += 1 + print(file=fout) + print('/* Block {block} of {blocks} */{comma}'.format(**locals()), file=fout) + print('\"{}\"'.format(line.strip()), file=fout) + print('/* Sentinel */, \"\"', file=fout) + + +def _embed_bytefile(fin, fout, is_text): + """ + Format a binary file for embedding in a C++ source file. + This version works directly with a single .pyc file. + """ + fname = fin.name + remark = ("No .pyc file because '--LIMITED-API=yes'" if is_text else + "The .pyc header is stripped away") + print(textwrap.dedent(""" + /* + * This is the file "{fname}" as a simple byte array. + * It can be directly embedded without any further processing. + * {remark}. + */ + """).format(**locals()).strip(), file=fout) + headsize = ( 0 if is_text else + 16 if sys.version_info >= (3, 7) else 12 if sys.version_info >= (3, 3) else 8) + binstr = fin.read()[headsize:] + if is_text: + try: + compile(binstr, fin.name, "exec") + except SyntaxError as e: + print(e) + traceback.print_exc(file=sys.stdout) + print(textwrap.dedent(""" + ************************************************************************* + *** + *** Could not compile the boot loader '{fname}'! + *** + ************************************************************************* + """).format(version=sys.version_info[:3], **locals())) + raise SystemError + else: + try: + marshal.loads(binstr) + except ValueError as e: + print(e) + traceback.print_exc(file=sys.stdout) + print(textwrap.dedent(""" + ************************************************************************* + *** + *** This Python version {version} seems to have a new .pyc header size. + *** Please correct the 'headsize' constant ({headsize}). + *** + ************************************************************************* + """).format(version=sys.version_info[:3], **locals())) + raise SystemError + + print(file=fout) + use_ord = sys.version_info[0] == 2 + for i in range(0, len(binstr), 16): + for c in bytes(binstr[i : i + 16]): + print("{:#4},".format(ord(c) if use_ord else c), file=fout, end="") + print(file=fout) + print("/* End Of File */", file=fout) + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--cmake-dir', nargs="?") + parser.add_argument('--limited-api', type=str2bool) + args = parser.parse_args() + if args.cmake_dir: + work_dir = os.path.abspath(args.cmake_dir) + create_zipfile(args.limited_api) diff --git a/sources/shiboken2/libshiboken/embed/module_collector.py b/sources/shiboken2/libshiboken/embed/module_collector.py new file mode 100644 index 000000000..3eaa0be5d --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/module_collector.py @@ -0,0 +1,105 @@ +# This Python file uses the following encoding: utf-8 +# It has been edited by fix-complaints.py . + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of Qt for Python. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +module_collector.py + +Collect a number of modules listed on the command line. + +The purpose of this script is to generate the scripts needed for +a complete isolation of the signature extension. + +Usage: + +Run this script in one of the used python versions. +It will create an executable archive of the files on the command line. +""" + +import sys +import os +import argparse +import pickle +from textwrap import dedent + +def source_archive(module, modname): + fname = os.path.splitext(module.__file__)[0] + ".py" + with open(fname) as source: + text = source.read() + encoded = text.replace("'''", "(triple_single)") + # modname = module.__name__ + # Do not use: Some modules rename themselves! + version = ".".join(map(str, sys.version_info[:3])) + shortname = os.path.basename(fname) + preamble = dedent(r""" + # BEGIN SOURCE ARCHIVE Python {version} module {modname} + + sources = {{}} if "sources" not in globals() else sources + sources["{modname}"] = '''\ + {encoded}'''.replace("(triple_single)", "'''") + + # END SOURCE ARCHIVE Python {version} module {modname} + """).format(**locals()) + return preamble + +def read_all(modules): + collected = "" + for modname in modules: + mod = __import__(modname) + collected += source_archive(mod, modname) + return collected + +def license_header(): + license = os.path.join(os.path.dirname(__file__), "qt_python_license.txt") + with open(license) as f: + return f.read() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('modules', nargs="+") + args = parser.parse_args() + print("modules:", args.modules) + ret = license_header() + read_all(args.modules) + ma_mi = "_".join(map(str, sys.version_info[:2])) + outpath = os.path.join(os.path.dirname(__file__), "..", "..", "shibokenmodule", + "files.dir", "shibokensupport", "python_minilib_{ma_mi}.py".format(**locals())) + with open(outpath, "w") as f: + f.write(ret) diff --git a/sources/shiboken2/libshiboken/embed/qt_python_license.txt b/sources/shiboken2/libshiboken/embed/qt_python_license.txt new file mode 100644 index 000000000..b5f8c581a --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/qt_python_license.txt @@ -0,0 +1,87 @@ +# This Python file uses the following encoding: utf-8 +# It has been edited by fix-complaints.py . + +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of Qt for Python. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +## +## PSF LICENSE AGREEMENT FOR PYTHON 3.7.0 +## +## 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and +## the Individual or Organization ("Licensee") accessing and otherwise using Python +## 3.7.0 software in source or binary form and its associated documentation. +## +## 2. Subject to the terms and conditions of this License Agreement, PSF hereby +## grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +## analyze, test, perform and/or display publicly, prepare derivative works, +## distribute, and otherwise use Python 3.7.0 alone or in any derivative +## version, provided, however, that PSF's License Agreement and PSF's notice of +## copyright, i.e., "Copyright © 2001-2018 Python Software Foundation; All Rights +## Reserved" are retained in Python 3.7.0 alone or in any derivative version +## prepared by Licensee. +## +## 3. In the event Licensee prepares a derivative work that is based on or +## incorporates Python 3.7.0 or any part thereof, and wants to make the +## derivative work available to others as provided herein, then Licensee hereby +## agrees to include in any such work a brief summary of the changes made to Python +## 3.7.0. +## +## 4. PSF is making Python 3.7.0 available to Licensee on an "AS IS" basis. +## PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF +## EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR +## WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE +## USE OF PYTHON 3.7.0 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. +## +## 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 3.7.0 +## FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF +## MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 3.7.0, OR ANY DERIVATIVE +## THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. +## +## 6. This License Agreement will automatically terminate upon a material breach of +## its terms and conditions. +## +## 7. Nothing in this License Agreement shall be deemed to create any relationship +## of agency, partnership, or joint venture between PSF and Licensee. This License +## Agreement does not grant permission to use PSF trademarks or trade name in a +## trademark sense to endorse or promote products or services of Licensee, or any +## third party. +## +## 8. By copying, installing or otherwise using Python 3.7.0, Licensee agrees +## to be bound by the terms and conditions of this License Agreement. +## diff --git a/sources/shiboken2/libshiboken/embed/signature_bootstrap.py b/sources/shiboken2/libshiboken/embed/signature_bootstrap.py new file mode 100644 index 000000000..6ce5ab95a --- /dev/null +++ b/sources/shiboken2/libshiboken/embed/signature_bootstrap.py @@ -0,0 +1,178 @@ +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +signature_bootstrap.py +---------------------- + +This file was originally directly embedded into the C source. +After it grew more and more, I now prefer to have it as Python file. + +Meanwhile, there is also no more a stub loader necessary: +Because we meanwhile have embedding support, we could also load this file +directly from a .pyc file. + +This file replaces the hard to read Python stub in 'signature.cpp', and we +could distinguish better between bootstrap related functions and loader +functions. +It is embedded into 'signature.cpp' as "embed/signature_bootstrap.inc". +""" + +from __future__ import print_function, absolute_import + +recursion_trap = 0 + +# We avoid real imports in phase 1 that could fail (simply removed all). +# Python 2 is not able to import when the extension import is still active. +# Phase 1 simply defines the functions, which will be used in Phase 2. + +def bootstrap(): + import sys + import os + import tempfile + import traceback + from contextlib import contextmanager + + global recursion_trap + if recursion_trap: + # we are probably called from outside, already + print("Recursion occurred in Bootstrap. Did you start by hand? Then it's ok.") + print("But you should trigger start by 'type.__signature__', only!") + recursion_trap += 1 + + @contextmanager + def ensure_shibokensupport(support_path): + # Make sure that we always have the shibokensupport containing package first. + # Also remove any prior loaded module of this name, just in case. + sys.path.insert(0, support_path) + + sbks = "shibokensupport" + if sbks in sys.modules: + del sys.modules[sbks] + prefix = sbks + "." + for key in list(key for key in sys.modules if key.startswith(prefix)): + del sys.modules[key] + try: + import shibokensupport + yield + except Exception as e: + print("Problem importing shibokensupport:") + print(e) + traceback.print_exc() + print("sys.path:") + for p in sys.path: + print(" " + p) + sys.stdout.flush() + sys.exit(-1) + sys.path.remove(support_path) + + try: + import shiboken2 as root + except ImportError: + # uninstalled case without ctest, try only this one which has __init__: + import shibokenmodule as root + rp = os.path.realpath(os.path.dirname(root.__file__)) + # This can be the shiboken2 directory or the binary module, so search. + look_for = "files.dir" + while len(rp) > 3 and not os.path.exists(os.path.join(rp, look_for)): + rp = os.path.abspath(os.path.join(rp, "..")) + + # Here we decide if we work embedded or not. + embedding_var = "pyside_uses_embedding" + use_embedding = bool(getattr(sys, embedding_var, False)) + # We keep the zip file for inspection if the sys variable has been set. + keep_zipfile = hasattr(sys, embedding_var) + real_dir = os.path.join(rp, look_for) + + # We report in sys what we used. We could put more here as well. + if not os.path.exists(real_dir): + use_embedding = True + support_path = prepare_zipfile() if use_embedding else real_dir + setattr(sys, embedding_var, use_embedding) + + try: + with ensure_shibokensupport(support_path): + from shibokensupport.signature import loader + + except Exception as e: + print('Exception:', e) + traceback.print_exc(file=sys.stdout) + + finally: + if use_embedding and not keep_zipfile: + # clear the temp zipfile + try: + os.remove(support_path) + except OSError as e: + print(e) + print("Error deleting {support_path}, ignored".format(**locals())) + return loader + +# New functionality: Loading from a zip archive. +# There exists the zip importer, but as it is written, only real zip files are +# supported. Before I will start an own implementation, it is easiest to use +# a temporary zip file. + +def prepare_zipfile(): + """ + Write the zip file to a real file and return its name. + It will be implicitly opened as such when we add the name to sys.path . + """ + import base64 + import tempfile + import os + import zipfile + + # 'zipstring_sequence' comes from signature.cpp + zipbytes = base64.b64decode(''.join(zipstring_sequence)) + fd, fname = tempfile.mkstemp(prefix='embedded.', suffix='.zip') + os.write(fd, zipbytes) + os.close(fd) + # Let us test the zipfile if it really is one. + # Otherwise, zipimporter would simply ignore it without notice. + try: + z = zipfile.ZipFile(fname) + z.close() + except zipfile.BadZipFile as e: + print('Broken Zip File:', e) + traceback.print_exc(file=sys.stdout) + finally: + return fname + +# eof |