diff options
Diffstat (limited to 'sources/shiboken6/libshiboken/signature')
6 files changed, 1968 insertions, 0 deletions
diff --git a/sources/shiboken6/libshiboken/signature/signature.cpp b/sources/shiboken6/libshiboken/signature/signature.cpp new file mode 100644 index 000000000..05d3cad2d --- /dev/null +++ b/sources/shiboken6/libshiboken/signature/signature.cpp @@ -0,0 +1,478 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +//////////////////////////////////////////////////////////////////////////// +// +// signature.cpp +// ------------- +// +// This is the main file of the signature module. +// It contains the most important functions and avoids confusion +// by moving many helper functions elsewhere. +// +// General documentation can be found in `signature_doc.rst`. +// + +#include "basewrapper.h" +#include "autodecref.h" +#include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" + +#include "signature_p.h" +#include <structmember.h> + +using namespace Shiboken; + +extern "C" +{ + +static PyObject *CreateSignature(PyObject *props, PyObject *key) +{ + /* + * Here is the new function to create all signatures. It simply calls + * into Python and creates a signature object directly. + * This is so much simpler than using all the attributes explicitly + * to support '_signature_is_functionlike()'. + */ + return PyObject_CallFunction(pyside_globals->create_signature_func, + const_cast<char *>("(OO)"), props, key); +} + +PyObject *GetClassOrModOf(PyObject *ob) +{ + /* + * Return the type or module of a function or type. + * The purpose is finally to use the name of the object. + */ + if (PyType_Check(ob)) { + // PySide-928: The type case must do refcounting like the others as well. + Py_INCREF(ob); + return ob; + } + if (PyType_IsSubtype(Py_TYPE(ob), &PyCFunction_Type)) + return _get_class_of_cf(ob); + if (Py_TYPE(ob) == PepStaticMethod_TypePtr) + return _get_class_of_sm(ob); + if (Py_TYPE(ob) == PepMethodDescr_TypePtr) + return _get_class_of_descr(ob); + if (Py_TYPE(ob) == &PyWrapperDescr_Type) + return _get_class_of_descr(ob); + Py_FatalError("unexpected type in GetClassOrModOf"); + return nullptr; +} + +PyObject *GetTypeKey(PyObject *ob) +{ + assert(PyType_Check(ob) || PyModule_Check(ob)); + /* + * Obtain a unique key using the module name and the type name. + * + * PYSIDE-1286: We use correct __module__ and __qualname__, now. + */ + AutoDecRef module_name(PyObject_GetAttr(ob, PyMagicName::module())); + if (module_name.isNull()) { + // We have no module_name because this is a module ;-) + PyErr_Clear(); + module_name.reset(PyObject_GetAttr(ob, PyMagicName::name())); + return Py_BuildValue("O", module_name.object()); + } + AutoDecRef class_name(_get_qualname(ob)); + if (class_name.isNull()) { + Py_FatalError("Signature: missing class name in GetTypeKey"); + return nullptr; + } + return Py_BuildValue("(OO)", module_name.object(), class_name.object()); +} + +static PyObject *empty_dict = nullptr; + +PyObject *TypeKey_to_PropsDict(PyObject *type_key, PyObject *obtype) +{ + PyObject *dict = PyDict_GetItem(pyside_globals->arg_dict, type_key); + if (dict == nullptr) { + if (empty_dict == nullptr) + empty_dict = PyDict_New(); + dict = empty_dict; + } + if (!PyDict_Check(dict)) + dict = PySide_BuildSignatureProps(type_key); + return dict; +} + +static PyObject *_GetSignature_Cached(PyObject *props, PyObject *func_kind, PyObject *modifier) +{ + // Special case: We want to know the func_kind. + if (modifier) { + PyUnicode_InternInPlace(&modifier); + if (modifier == PyMagicName::func_kind()) + return Py_BuildValue("O", func_kind); + } + + AutoDecRef key(modifier == nullptr ? Py_BuildValue("O", func_kind) + : Py_BuildValue("(OO)", func_kind, modifier)); + PyObject *value = PyDict_GetItem(props, key); + if (value == nullptr) { + // we need to compute a signature object + value = CreateSignature(props, key); + if (value != nullptr) { + if (PyDict_SetItem(props, key, value) < 0) + // this is an error + return nullptr; + } + else { + // key not found + Py_RETURN_NONE; + } + } + return Py_INCREF(value), value; +} + +PyObject *GetSignature_Function(PyObject *obfunc, PyObject *modifier) +{ + // make sure that we look into PyCFunction, only... + if (Py_TYPE(obfunc) == PepFunction_TypePtr) + Py_RETURN_NONE; + AutoDecRef obtype_mod(GetClassOrModOf(obfunc)); + AutoDecRef type_key(GetTypeKey(obtype_mod)); + if (type_key.isNull()) + Py_RETURN_NONE; + PyObject *dict = TypeKey_to_PropsDict(type_key, obtype_mod); + if (dict == nullptr) + return nullptr; + AutoDecRef func_name(PyObject_GetAttr(obfunc, PyMagicName::name())); + PyObject *props = !func_name.isNull() ? PyDict_GetItem(dict, func_name) : nullptr; + if (props == nullptr) + Py_RETURN_NONE; + + int flags = PyCFunction_GET_FLAGS(obfunc); + PyObject *func_kind; + if (PyModule_Check(obtype_mod)) + func_kind = PyName::function(); + else if (flags & METH_CLASS) + func_kind = PyName::classmethod(); + else if (flags & METH_STATIC) + func_kind = PyName::staticmethod(); + else + func_kind = PyName::method(); + return _GetSignature_Cached(props, func_kind, modifier); +} + +PyObject *GetSignature_Wrapper(PyObject *ob, PyObject *modifier) +{ + AutoDecRef func_name(PyObject_GetAttr(ob, PyMagicName::name())); + AutoDecRef objclass(PyObject_GetAttr(ob, PyMagicName::objclass())); + AutoDecRef class_key(GetTypeKey(objclass)); + if (func_name.isNull() || objclass.isNull() || class_key.isNull()) + return nullptr; + PyObject *dict = TypeKey_to_PropsDict(class_key, objclass); + if (dict == nullptr) + return nullptr; + PyObject *props = PyDict_GetItem(dict, func_name); + if (props == nullptr) + Py_RETURN_NONE; + return _GetSignature_Cached(props, PyName::method(), modifier); +} + +PyObject *GetSignature_TypeMod(PyObject *ob, PyObject *modifier) +{ + AutoDecRef ob_name(PyObject_GetAttr(ob, PyMagicName::name())); + AutoDecRef ob_key(GetTypeKey(ob)); + + PyObject *dict = TypeKey_to_PropsDict(ob_key, ob); + if (dict == nullptr) + return nullptr; + PyObject *props = PyDict_GetItem(dict, ob_name); + if (props == nullptr) + Py_RETURN_NONE; + return _GetSignature_Cached(props, PyName::method(), modifier); +} + +//////////////////////////////////////////////////////////////////////////// +// +// get_signature -- providing a superior interface +// +// Additional to the interface via `__signature__`, we also provide +// a general function, which allows for different signature layouts. +// The `modifier` argument is a string that is passed in from `loader.py`. +// Configuration what the modifiers mean is completely in Python. +// + +PyObject *get_signature_intern(PyObject *ob, PyObject *modifier) +{ + if (PyType_IsSubtype(Py_TYPE(ob), &PyCFunction_Type)) + return pyside_cf_get___signature__(ob, modifier); + if (Py_TYPE(ob) == PepStaticMethod_TypePtr) + return pyside_sm_get___signature__(ob, modifier); + if (Py_TYPE(ob) == PepMethodDescr_TypePtr) + return pyside_md_get___signature__(ob, modifier); + if (PyType_Check(ob)) + return pyside_tp_get___signature__(ob, modifier); + if (Py_TYPE(ob) == &PyWrapperDescr_Type) + return pyside_wd_get___signature__(ob, modifier); + return nullptr; +} + +static PyObject *get_signature(PyObject * /* self */, PyObject *args) +{ + PyObject *ob; + PyObject *modifier = nullptr; + + init_module_1(); + + if (!PyArg_ParseTuple(args, "O|O", &ob, &modifier)) + return nullptr; + if (Py_TYPE(ob) == PepFunction_TypePtr) + Py_RETURN_NONE; + PyObject *ret = get_signature_intern(ob, modifier); + if (ret != nullptr) + return ret; + Py_RETURN_NONE; +} + +PyMethodDef signature_methods[] = { + {"get_signature", (PyCFunction)get_signature, METH_VARARGS, + "get the __signature__, but pass an optional string parameter"}, + {nullptr, nullptr} +}; + +//////////////////////////////////////////////////////////////////////////// +// +// Argument Handling +// ----------------- +// +// * PySide_BuildSignatureArgs +// +// Called during class or module initialization. +// The signature strings from the C modules are stored in a dict for +// later use. +// +// * PySide_BuildSignatureProps +// +// Called on demand during signature retieval. This function calls all the way +// through `parser.py` and prepares all properties for the functions of the class. +// The parsed properties can then be used to create signature objects. +// + +static int PySide_BuildSignatureArgs(PyObject *obtype_mod, const char *signatures[]) +{ + init_module_1(); + AutoDecRef type_key(GetTypeKey(obtype_mod)); + /* + * PYSIDE-996: Avoid string overflow in MSVC, which has a limit of + * 2**15 unicode characters (64 K memory). + * Instead of one huge string, we take a ssize_t that is the + * address of a string array. It will not be turned into a real + * string list until really used by Python. This is quite optimal. + */ + AutoDecRef numkey(Py_BuildValue("n", signatures)); + if (type_key.isNull() || numkey.isNull() + || PyDict_SetItem(pyside_globals->arg_dict, type_key, numkey) < 0) + return -1; + /* + * We record also a mapping from type key to type/module. This helps to + * lazily initialize the Py_LIMITED_API in name_key_to_func(). + */ + return PyDict_SetItem(pyside_globals->map_dict, type_key, obtype_mod) == 0 ? 0 : -1; +} + +PyObject *PySide_BuildSignatureProps(PyObject *type_key) +{ + /* + * Here is the second part of the function. + * This part will be called on-demand when needed by some attribute. + * We simply pick up the arguments that we stored here and replace + * them by the function result. + */ + init_module_2(); + if (type_key == nullptr) + return nullptr; + PyObject *numkey = PyDict_GetItem(pyside_globals->arg_dict, type_key); + AutoDecRef strings(_address_to_stringlist(numkey)); + if (strings.isNull()) + return nullptr; + AutoDecRef arg_tup(Py_BuildValue("(OO)", type_key, strings.object())); + if (arg_tup.isNull()) + return nullptr; + PyObject *dict = PyObject_CallObject(pyside_globals->pyside_type_init_func, arg_tup); + if (dict == nullptr) { + if (PyErr_Occurred()) + return nullptr; + // No error: return an empty dict. + if (empty_dict == nullptr) + empty_dict = PyDict_New(); + return empty_dict; + } + // PYSIDE-1019: Build snake case versions of the functions. + if (insert_snake_case_variants(dict) < 0) + return nullptr; + // We replace the arguments by the result dict. + if (PyDict_SetItem(pyside_globals->arg_dict, type_key, dict) < 0) + return nullptr; + return dict; +} +// +//////////////////////////////////////////////////////////////////////////// + +static int PySide_FinishSignatures(PyObject *module, const char *signatures[]) +{ + /* + * Initialization of module functions and resolving of static methods. + */ + const char *name = PyModule_GetName(module); + if (name == nullptr) + return -1; + + // we abuse the call for types, since they both have a __name__ attribute. + if (PySide_BuildSignatureArgs(module, signatures) < 0) + return -1; + + /* + * Note: This function crashed when called from PySide_BuildSignatureArgs. + * Probably this was an import timing problem. + * + * Pep384: We need to switch this always on since we have no access + * to the PyCFunction attributes. Therefore I simplified things + * and always use our own mapping. + */ + PyObject *key, *func, *obdict = PyModule_GetDict(module); + Py_ssize_t pos = 0; + + while (PyDict_Next(obdict, &pos, &key, &func)) + if (PyCFunction_Check(func)) + if (PyDict_SetItem(pyside_globals->map_dict, func, module) < 0) + return -1; + if (_finish_nested_classes(obdict) < 0) + return -1; + // The finish_import function will not work the first time since phase 2 + // was not yet run. But that is ok, because the first import is always for + // the shiboken module (or a test module). + if (pyside_globals->finish_import_func == nullptr) { + assert(strncmp(name, "PySide2.", 8) != 0); + return 0; + } + AutoDecRef ret(PyObject_CallFunction( + pyside_globals->finish_import_func, const_cast<char *>("(O)"), module)); + return ret.isNull() ? -1 : 0; +} + +//////////////////////////////////////////////////////////////////////////// +// +// External functions interface +// +// These are exactly the supported functions from `signature.h`. +// + +int InitSignatureStrings(PyTypeObject *type, const char *signatures[]) +{ + auto *ob_type = reinterpret_cast<PyObject *>(type); + int ret = PySide_BuildSignatureArgs(ob_type, signatures); + if (ret < 0) { + PyErr_Print(); + PyErr_SetNone(PyExc_ImportError); + } + return ret; +} + +void FinishSignatureInitialization(PyObject *module, const char *signatures[]) +{ + /* + * This function is called at the very end of a module initialization. + * We now patch certain types to support the __signature__ attribute, + * initialize module functions and resolve static methods. + * + * Still, it is not possible to call init phase 2 from here, + * because the import is still running. Do it from Python! + */ + if ( PySide_PatchTypes() < 0 + || PySide_FinishSignatures(module, signatures) < 0) { + PyErr_Print(); + PyErr_SetNone(PyExc_ImportError); + } +} + +void SetError_Argument(PyObject *args, const char *func_name) +{ + /* + * This function replaces the type error construction with extra + * overloads parameter in favor of using the signature module. + * Error messages are rare, so we do it completely in Python. + */ + init_module_1(); + init_module_2(); + AutoDecRef res(PyObject_CallFunction(pyside_globals->seterror_argument_func, + const_cast<char *>("(Os)"), args, func_name)); + if (res.isNull()) { + PyErr_Print(); + Py_FatalError("seterror_argument did not receive a result"); + } + PyObject *err, *msg; + if (!PyArg_UnpackTuple(res, func_name, 2, 2, &err, &msg)) { + PyErr_Print(); + Py_FatalError("unexpected failure in seterror_argument"); + } + PyErr_SetObject(err, msg); +} + +/* + * Support for the metatype SbkObjectType_Type's tp_getset. + * + * This was not necessary for __signature__, because PyType_Type inherited it. + * But the __doc__ attribute existed already by inheritance, and calling + * PyType_Modified() is not supported. So we added the getsets explicitly + * to the metatype. + */ + +PyObject *Sbk_TypeGet___signature__(PyObject *ob, PyObject *modifier) +{ + return pyside_tp_get___signature__(ob, modifier); +} + +PyObject *Sbk_TypeGet___doc__(PyObject *ob) +{ + return pyside_tp_get___doc__(ob); +} + +PyObject *GetFeatureDict() +{ + init_module_1(); + return pyside_globals->feature_dict; +} + +} //extern "C" diff --git a/sources/shiboken6/libshiboken/signature/signature_doc.rst b/sources/shiboken6/libshiboken/signature/signature_doc.rst new file mode 100644 index 000000000..95f0b8b08 --- /dev/null +++ b/sources/shiboken6/libshiboken/signature/signature_doc.rst @@ -0,0 +1,357 @@ +************************* +The signature C extension +************************* + +This module is a C extension for CPython 3.5 and up, and CPython 2.7. +Its purpose is to provide support for the ``__signature__`` attribute +of builtin PyCFunction objects. + + +Short Introduction to the Topic +=============================== + +Beginning with CPython 3.5, Python functions began to grow a ``__signature__`` +attribute for normal Python functions. This is totally optional and just +a nice-to-have feature in Python. + +PySide, on the other hand, could use ``__signature__`` very much, because the +typing info for the 15000+ PySide functions is really missing, and it +would be nice to have this info directly available. + + +The Idea to Support Signatures +============================== + +We want to have an additional ``__signature__`` attribute in all PySide +methods, without changing lots of generated code. +Therefore, we did not change any of the existing data structures, +but supported the new attribute by a global dictionary. + +When the ``__signature__`` property is requested, a method is called that +does a lookup in the global dict. This is a flexible approach with little impact +to the rest of the project. It has very limited overhead compared to direct +attribute access, but for the need of a signature access from time to time, +this is an adequate compromise. + + +How this Code Works +------------------- + +Signatures are supported for regular Python functions, only. Creating signatures +for ``PyCFunction`` objects would require quite some extra effort in Python. + +Fortunately, we found this special *stealth* technique, that saves us most of the +needed effort: + +The basic idea is to create a dummy Python function with **varnames**, **defaults** +and **annotations** properties, and then to use the inspect +module to create a signature object. This object is returned as the computed +result of the ``__signature__`` attribute of the real ``PyCFunction`` object. + +There is one thing that really changes Python a bit: + +* We added the ``__signature__`` attribute to every function. + +That is a little change to Python that does not harm, but it saves us +tons of code, that was needed in the early versions of the module. + +The internal work is done in two steps: + +* All functions of a class get the *signature text* when the module is imported. + This is only a very small overhead added to the startup time. It is a single + string for each whole class. +* The actual signature object is created later, when the attribute is really + requested. Signatures are cached and only created on first access. + +Example: + +The ``PyCFunction`` ``QtWidgets.QApplication.palette`` is interrogated for its +signature. That means ``pyside_sm_get___signature__()`` is called. +It calls ``GetSignature_Function`` which returns the signature if it is found. + + +Why this Code is Fast +--------------------- + +It costs a little time (maybe 6 seconds) to run through every single signature +object, since these are more than 25000 Python objects. But all the signature +objects will be rarely accessed but in special applications. +The normal case are only a few accesses, and these are working pretty fast. + +The key to make this signature module fast is to avoid computation as much as +possible. When no signature objects are used, then almost no time is lost in +initialization. Only the above mentioned strings and some support modules are +additionally loaded on ``import PySide2``. +When it comes to signature usage, then late initialization is used and cached. +This technique is also known as *full laziness* in haskell. + +There are actually two locations where late initialization occurs: + +* ``dict`` can be no dict but a tuple. That is the initial argument tuple that + was saved by ``PySide_BuildSignatureArgs`` at module load time. + If so, then ``pyside_type_init`` in parser.py will be called, + which parses the string and creates the dict. +* ``props`` can be empty. Then ``create_signature`` in loader.py + is called, which uses a dummy function to produce a signature instance + with the inspect module. + +The initialization that is always done is just two dictionary writes +per class, and we have about 1000 classes. +To measure the additional overhead, we have simulated what happens +when ``from PySide2 import *`` is performed. +It turned out that the overhead is below 0.5 ms. + + +The Signature Package Structure +------------------------------- + +The C++ code involved with the signature module is completely in the file +shiboken6/libshiboken/signature.cpp . All other functionality is implemented in +the ``signature`` Python package. It has the following structure:: + + shiboken6/files.dir/shibokensupport/ + backport_inspect.py + + signature/ + loader.py + parser.py + mapping.py + errorhandler.py + layout.py + + lib/ + enum_sig.py + tool.py + + + +Really important are the **parser**, **mapping**, **errorhandler**, **enum_sig**, +**layout** and **loader** modules. The rest is needed to create Python 2 compatibility +or be compatible with embedding and installers. + + +loader.py +~~~~~~~~~ + +This module assembles and imports the ``inspect`` module, and then exports the +``create_signature`` function. This function takes a fake function and some +attributes and builds a ``__signature__`` object with the inspect module. + + +parser.py +~~~~~~~~~ + +This module takes a class signatures string from C++ and parses it into the +needed properties for the ``create_signature`` function. Its entry point is the +``pyside_type_init`` function, which is called from the C module via ``loader.py``. + + +mapping.py +~~~~~~~~~~ + +The purpose of the mapping module is maintaining a list of replacement strings +that map from the *signature text* in C to the property strings that Python +needs. A lot of mappings are resolved by rather complex expressions in ``parser.py``, +but a few hundred cases are better to spell explicitly, here. + + +errorhandler.py +~~~~~~~~~~~~~~~ + +Since ``Qt For Python 5.12``, we no longer use the builtin type error messages from C++. +Instead, we get much better results with the signature module. At the same time, +this enforced supporting shiboken as well, and the signature module was no longer +optional. + + +enum_sig.py +~~~~~~~~~~~ + +The diverse applications of the signature module all needed to iterate over modules, +classes and functions. In order to centralize this enumeration, the process has +been factored out as a context manager. The user has only to supply functions +that do the actual formatting. + +See for example the .pyi generator ``pyside2/PySide2/support/generate_pyi.py``. + + +layout.py +~~~~~~~~~ + +As more applications used the signature module, different formatting of signatures +was needed. To support that, we created the function ``create_signature``, which +has a parameter to choose from some prefefined layouts. + + +*typing27.py* +~~~~~~~~~~~~~ + +Python 2 has no typing module at all. This is a backport of the minimum that is needed. + + +*backport_inspect.py* +~~~~~~~~~~~~~~~~~~~~~ + +Python 2 has an inspect module, but lacks the signature functions, completely. +This module adds the missing functionality, which is merged at runtime into +the inspect module. + + +Multiple Arities +---------------- + +One aspect that was ignored so far was *multiple arities*: How to handle it when +a function has more than one signature? + +I did not find any note on how multiple signatures should be treated in Python, +but this simple rules seem to work well: + +* If there is a list, then it is a multi-signature. +* Otherwise, it is a simple signature. + + +Impacts of The Signature Module +=============================== + +The signature module has a number of impacts to other PySide modules, which were +created as a consequence of its existence, and there will be a few more in the +future: + + +existence_test.py +----------------- + +The file ``pyside2/tests/registry/existence_test.py`` was written using the +signatures from the signatures module. The idea is that there are some 15000 +functions with a certain signature. + +These functions should not get lost by some bad check-in. Therefore, a list +of all existing signatures is kept as a module that assembles a +dictionary. The function existence is checked, and also the exact arity. + +This module exists for every PySide release and every platform. The initial +module is generated once and saved as ``exists_{plat}_{version}.py``. + +An error is normally only reported as a warning, but: + + +Interaction With The Coin Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When this test program is run in COIN, then the warnings are turned into +errors. The reason is that only in COIN, we have a stable configuration +of PySide modules that can reliably be compared. + +These modules have the name ``exists_{platf}_{version}_ci.py``, and as a big +exception for generated code, these files are *intentionally* checked in. + + +What Happens When a List is Missing? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a new version of PySide gets created, then the existence test files +initially do not exist. + +When a COIN test is run, then it will complain about the error and create +the missing module on standard output. +But since COIN tests are run multiple times, the output that was generated +by the first test will still exist at the subsequent runs. +(If COIN was properly implemented, we could not take that advantage and +would need to implement that as an extra exception.) + +As a result, a missing module will be reported as a test which partially +succeeded (called "FLAKY"). To avoid further flaky tests and to activate as a real test, +we can now capture the error output of COIN and check the generated module +in. + + +Explicitly Enforcing Recreation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The former way to regenerate the registry files was to remove the files +and check that in. This has the desired effect, but creates huge deltas. +As a more efficient way, we have prepared a comment in the first line +that contains the word "recreate". +By uncommenting this line, a NameError is triggered, which has the same +effect. + + +init_platform.py +~~~~~~~~~~~~~~~~ + +For generating the ``exists_{platf}_{version}`` modules, the module +``pyside2/tests/registry/init_platform.py`` was written. It can be used +standalone from the commandline, to check the compatibility of some +changes, directly. + + +scrape_testresults.py +--------------------- + +To simplify and automate the process of extracting the ``exists_{platf}_{version}_ci.py`` +files, the script ``pyside2/tests/registry/scrape_testresults.py`` has been written. + +This script scans the whole testresults website for PySide, that is:: + + https://testresults.qt.io/coin/api/results/pyside/pyside-setup/ + +On the first scan, the script runs less than 30 minutes. After that, a cache +is generated and the scan works *much* faster. The test results are placed +into the folder ``pyside2/tests/registry/testresults/embedded/`` with a +unique name that allows for easy sorting. Example:: + + testresults/embedded/2018_09_10_10_40_34-test_1536891759-exists_linux_5_11_2_ci.py + +These files are created only once. If they already exist, they are not touched, again. +The file `pyside2/tests/registry/known_urls.json`` holds all scanned URLs after +a successful scan. The ``testresults/embedded`` folder can be kept for reference +or can be removed. Important is only the json file. + +The result of a scan is then directly placed into the ``pyside2/tests/registry/`` +folder. It should be reviewed and then eventually checked in. + + +generate_pyi.py +--------------- + +``pyside2/PySide2/support/generate_pyi.py`` is still under development. +This module generates so-called hinting stubs for integration of PySide +with diverse *Python IDEs*. + +Although this module creates the stubs as an add-on, the +impact on the quality of the signature module is considerable: + +The module must create syntactically correct ``.pyi`` files which contain +not only signatures but also constants and enums of all PySide modules. +This serves as an extra challenge that has a very positive effect on +the completeness and correctness of signatures. + + +Current Extensions +------------------ + +Before the signature module was written, there already existed the concept of +signatures, but in a more C++ - centric way. From that time, there existed +the error messages, which are created when a function gets wrong argument types. + +These error messages were replaced by text generated on demand by +the signature module, in order to be more consistent and correct. +This was implemented in ``Qt For Python 5.12.0``. + +Additionally, the ``__doc__`` attribute of PySide methods was not set. +It was easy to get a nice ``help()`` feature by creating signatures +as default content for docstrings. +This was implemented in ``Qt For Python 5.12.1``. + + +Literature +========== + + `PEP 362 – Function Signature Object <https://www.python.org/dev/peps/pep-0362/>`__ + + `PEP 484 – Type Hints <https://www.python.org/dev/peps/pep-0484/>`__ + + `PEP 3107 – Function Annotations <https://www.python.org/dev/peps/pep-3107/>`__ + + +*Personal Remark: This module is dedicated to our lovebird "Püppi", who died on 2017-09-15.* diff --git a/sources/shiboken6/libshiboken/signature/signature_extend.cpp b/sources/shiboken6/libshiboken/signature/signature_extend.cpp new file mode 100644 index 000000000..1490a6003 --- /dev/null +++ b/sources/shiboken6/libshiboken/signature/signature_extend.cpp @@ -0,0 +1,294 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +//////////////////////////////////////////////////////////////////////////// +// +// signature_extend.cpp +// -------------------- +// +// This file contains the additions and changes to the following +// Python types: +// +// PyMethodDescr_Type +// PyCFunction_Type +// PyStaticMethod_Type +// PyType_Type +// PyWrapperDescr_Type +// +// Their `tp_getset` fields are modified so support the `__signature__` +// attribute and additions to the `__doc__` attribute. +// + +#include "autodecref.h" +#include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" + +#include "signature_p.h" + +using namespace Shiboken; + +extern "C" { + +typedef PyObject *(*signaturefunc)(PyObject *, PyObject *); + +static PyObject *_get_written_signature(signaturefunc sf, PyObject *ob, PyObject *modifier) +{ + /* + * Be a writable Attribute, but have a computed value. + * + * If a signature has not been written, call the signature function. + * If it has been written, return the written value. + * After __del__ was called, the function value re-appears. + * + * Note: This serves also for the new version that does not allow any + * assignment if we have a computed value. We only need to check if + * a computed value exists and then forbid writing. + * See pyside_set___signature + */ + PyObject *ret = PyDict_GetItem(pyside_globals->value_dict, ob); + if (ret == nullptr) + return ob == nullptr ? nullptr : sf(ob, modifier); + Py_INCREF(ret); + return ret; +} + +PyObject *pyside_cf_get___signature__(PyObject *func, PyObject *modifier) +{ + init_module_2(); + return _get_written_signature(GetSignature_Function, func, modifier); +} + +PyObject *pyside_sm_get___signature__(PyObject *sm, PyObject *modifier) +{ + init_module_2(); + AutoDecRef func(PyObject_GetAttr(sm, PyMagicName::func())); + if (Py_TYPE(func) == PepFunction_TypePtr) + return PyObject_GetAttr(func, PyMagicName::signature()); + return _get_written_signature(GetSignature_Function, func, modifier); +} + +PyObject *pyside_md_get___signature__(PyObject *ob_md, PyObject *modifier) +{ + init_module_2(); + AutoDecRef func(name_key_to_func(ob_md)); + if (func.object() == Py_None) + return Py_None; + if (func.isNull()) + Py_FatalError("missing mapping in MethodDescriptor"); + return pyside_cf_get___signature__(func, modifier); +} + +PyObject *pyside_wd_get___signature__(PyObject *ob, PyObject *modifier) +{ + init_module_2(); + return _get_written_signature(GetSignature_Wrapper, ob, modifier); +} + +PyObject *pyside_tp_get___signature__(PyObject *obtype_mod, PyObject *modifier) +{ + init_module_2(); + return _get_written_signature(GetSignature_TypeMod, obtype_mod, modifier); +} + +//////////////////////////////////////////////////////////////////////////// +// +// Augmenting builtin types with a __signature__ attribute. +// +// This is a harmless change to Python, similar like __text_signature__. +// We could avoid it, but then we would need to copy quite some module +// initialization functions which are pretty version- and word size +// dependent. I think this little patch is the lesser of the two evils. +// +// Please note that in fact we are modifying 'type', the metaclass of all +// objects, because we add new functionality. +// +// Addendum 2019-01-12: We now also compute a docstring from the signature. +// + +// keep the original __doc__ functions +static PyObject *old_cf_doc_descr = nullptr; +static PyObject *old_sm_doc_descr = nullptr; +static PyObject *old_md_doc_descr = nullptr; +static PyObject *old_tp_doc_descr = nullptr; +static PyObject *old_wd_doc_descr = nullptr; + +static int handle_doc_in_progress = 0; + +static PyObject *handle_doc(PyObject *ob, PyObject *old_descr) +{ + init_module_1(); + init_module_2(); + AutoDecRef ob_type_mod(GetClassOrModOf(ob)); + const char *name; + if (PyModule_Check(ob_type_mod)) + name = PyModule_GetName(ob_type_mod); + else + name = reinterpret_cast<PyTypeObject *>(ob_type_mod.object())->tp_name; + if (handle_doc_in_progress || name == nullptr + || strncmp(name, "PySide2.", 8) != 0) + return PyObject_CallMethodObjArgs(old_descr, + PyMagicName::get(), + ob, nullptr); + handle_doc_in_progress++; + PyObject *res = PyObject_CallFunction( + pyside_globals->make_helptext_func, + const_cast<char *>("(O)"), ob); + handle_doc_in_progress--; + if (res == nullptr) { + PyErr_Print(); + Py_FatalError("handle_doc did not receive a result"); + } + return res; +} + +static PyObject *pyside_cf_get___doc__(PyObject *cf) +{ + return handle_doc(cf, old_cf_doc_descr); +} + +static PyObject *pyside_sm_get___doc__(PyObject *sm) +{ + return handle_doc(sm, old_sm_doc_descr); +} + +static PyObject *pyside_md_get___doc__(PyObject *md) +{ + return handle_doc(md, old_md_doc_descr); +} + +PyObject *pyside_tp_get___doc__(PyObject *tp) +{ + return handle_doc(tp, old_tp_doc_descr); +} + +static PyObject *pyside_wd_get___doc__(PyObject *wd) +{ + return handle_doc(wd, old_wd_doc_descr); +} + +// the default setter for all objects +static int pyside_set___signature__(PyObject *op, PyObject *value) +{ + // By this additional check, this function refuses write access. + // We consider both nullptr and Py_None as not been written. + AutoDecRef has_val(get_signature_intern(op, nullptr)); + if (!(has_val.isNull() || has_val == Py_None)) { + PyErr_Format(PyExc_AttributeError, + "Attribute '__signature__' of '%.50s' object is not writable", + Py_TYPE(op)->tp_name); + return -1; + } + int ret = value == nullptr ? PyDict_DelItem(pyside_globals->value_dict, op) + : PyDict_SetItem(pyside_globals->value_dict, op, value); + Py_XINCREF(value); + return ret; +} + +static PyGetSetDef new_PyCFunction_getsets[] = { + {const_cast<char *>("__doc__"), (getter)pyside_cf_get___doc__}, + {const_cast<char *>("__signature__"), (getter)pyside_cf_get___signature__, + (setter)pyside_set___signature__}, + {nullptr} +}; + +static PyGetSetDef new_PyStaticMethod_getsets[] = { + {const_cast<char *>("__doc__"), (getter)pyside_sm_get___doc__}, + {const_cast<char *>("__signature__"), (getter)pyside_sm_get___signature__, + (setter)pyside_set___signature__}, + {nullptr} +}; + +static PyGetSetDef new_PyMethodDescr_getsets[] = { + {const_cast<char *>("__doc__"), (getter)pyside_md_get___doc__}, + {const_cast<char *>("__signature__"), (getter)pyside_md_get___signature__, + (setter)pyside_set___signature__}, + {nullptr} +}; + +static PyGetSetDef new_PyType_getsets[] = { + {const_cast<char *>("__doc__"), (getter)pyside_tp_get___doc__}, + {const_cast<char *>("__signature__"), (getter)pyside_tp_get___signature__, + (setter)pyside_set___signature__}, + {nullptr} +}; + +static PyGetSetDef new_PyWrapperDescr_getsets[] = { + {const_cast<char *>("__doc__"), (getter)pyside_wd_get___doc__}, + {const_cast<char *>("__signature__"), (getter)pyside_wd_get___signature__, + (setter)pyside_set___signature__}, + {nullptr} +}; + +int PySide_PatchTypes(void) +{ + static int init_done = 0; + + if (!init_done) { + AutoDecRef meth_descr(PyObject_GetAttrString( + reinterpret_cast<PyObject *>(&PyString_Type), "split")); + AutoDecRef wrap_descr(PyObject_GetAttrString( + reinterpret_cast<PyObject *>(Py_TYPE(Py_True)), "__add__")); + // abbreviations for readability + auto md_gs = new_PyMethodDescr_getsets; + auto md_doc = &old_md_doc_descr; + auto cf_gs = new_PyCFunction_getsets; + auto cf_doc = &old_cf_doc_descr; + auto sm_gs = new_PyStaticMethod_getsets; + auto sm_doc = &old_sm_doc_descr; + auto tp_gs = new_PyType_getsets; + auto tp_doc = &old_tp_doc_descr; + auto wd_gs = new_PyWrapperDescr_getsets; + auto wd_doc = &old_wd_doc_descr; + + if (meth_descr.isNull() || wrap_descr.isNull() + || PyType_Ready(Py_TYPE(meth_descr)) < 0 + || add_more_getsets(PepMethodDescr_TypePtr, md_gs, md_doc) < 0 + || add_more_getsets(&PyCFunction_Type, cf_gs, cf_doc) < 0 + || add_more_getsets(PepStaticMethod_TypePtr, sm_gs, sm_doc) < 0 + || add_more_getsets(&PyType_Type, tp_gs, tp_doc) < 0 + || add_more_getsets(Py_TYPE(wrap_descr), wd_gs, wd_doc) < 0 + ) + return -1; + init_done = 1; + } + return 0; +} + +} // extern "C" diff --git a/sources/shiboken6/libshiboken/signature/signature_globals.cpp b/sources/shiboken6/libshiboken/signature/signature_globals.cpp new file mode 100644 index 000000000..6af64682e --- /dev/null +++ b/sources/shiboken6/libshiboken/signature/signature_globals.cpp @@ -0,0 +1,295 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +//////////////////////////////////////////////////////////////////////////// +// +// signature_global.cpp +// +// This file contains the global data structures and init code. +// + +#include "autodecref.h" +#include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" + +#include "signature_p.h" + +using namespace Shiboken; + +extern "C" { + +static const char *PySide_CompressedSignaturePackage[] = { +#include "embed/signature_inc.h" + }; + +static const unsigned char PySide_SignatureLoader[] = { +#include "embed/signature_bootstrap_inc.h" + }; + +static PyObject *_init_pyside_extension(PyObject * /* self */, PyObject * /* args */) +{ + init_module_1(); + init_module_2(); + Py_RETURN_NONE; +} + +// This function will be inserted into __builtins__. +static PyMethodDef init_methods[] = { + {"_init_pyside_extension", (PyCFunction)_init_pyside_extension, METH_NOARGS}, + {nullptr, nullptr} +}; + +static safe_globals_struc *init_phase_1(PyMethodDef *init_meth) +{ + { + auto *p = reinterpret_cast<safe_globals_struc *> + (malloc(sizeof(safe_globals_struc))); + if (p == nullptr) + goto error; + /* + * Initializing module signature_bootstrap. + * Since we now have an embedding script, we can do this without any + * Python strings in the C code. + */ +#ifdef Py_LIMITED_API + // We must work for multiple versions, so use source code. +#else + AutoDecRef marshal_module(PyImport_Import(PyName::marshal())); + if (marshal_module.isNull()) + goto error; + AutoDecRef loads(PyObject_GetAttr(marshal_module, PyName::loads())); + if (loads.isNull()) + goto error; +#endif + char *bytes_cast = reinterpret_cast<char *>( + const_cast<unsigned char *>(PySide_SignatureLoader)); + AutoDecRef bytes(PyBytes_FromStringAndSize(bytes_cast, sizeof(PySide_SignatureLoader))); + if (bytes.isNull()) + goto error; +#ifdef Py_LIMITED_API + PyObject *builtins = PyEval_GetBuiltins(); + PyObject *compile = PyDict_GetItem(builtins, PyName::compile()); + if (compile == nullptr) + goto error; + AutoDecRef code_obj(PyObject_CallFunction(compile, "Oss", + bytes.object(), "(builtin)", "exec")); +#else + AutoDecRef code_obj(PyObject_CallFunctionObjArgs( + loads, bytes.object(), nullptr)); +#endif + if (code_obj.isNull()) + goto error; + p->helper_module = PyImport_ExecCodeModule(const_cast<char *> + ("signature_bootstrap"), code_obj); + if (p->helper_module == nullptr) + goto error; + // Initialize the module + PyObject *mdict = PyModule_GetDict(p->helper_module); + if (PyDict_SetItem(mdict, PyMagicName::builtins(), PyEval_GetBuiltins()) < 0) + goto error; + /* + * Unpack an embedded ZIP file with more signature modules. + * They will be loaded later with the zipimporter. + * Due to MSVC's limitation to 64k strings, we need to assemble pieces. + */ + const char **block_ptr = (const char **)PySide_CompressedSignaturePackage; + int npieces = 0; + PyObject *piece, *zipped_string_sequence = PyList_New(0); + if (zipped_string_sequence == nullptr) + return nullptr; + for (; **block_ptr != 0; ++block_ptr) { + npieces++; + // we avoid the string/unicode dilemma by not using PyString_XXX: + piece = Py_BuildValue("s", *block_ptr); + if (piece == nullptr || PyList_Append(zipped_string_sequence, piece) < 0) + goto error; + } + if (PyDict_SetItemString(mdict, "zipstring_sequence", zipped_string_sequence) < 0) + goto error; + Py_DECREF(zipped_string_sequence); + + // build a dict for diverse mappings + p->map_dict = PyDict_New(); + if (p->map_dict == nullptr) + goto error; + + // build a dict for the prepared arguments + p->arg_dict = PyDict_New(); + if (p->arg_dict == nullptr + || PyObject_SetAttrString(p->helper_module, "pyside_arg_dict", p->arg_dict) < 0) + goto error; + + // build a dict for assigned signature values + p->value_dict = PyDict_New(); + if (p->value_dict == nullptr) + goto error; + + // PYSIDE-1019: build a __feature__ dict + p->feature_dict = PyDict_New(); + if (p->feature_dict == nullptr + || PyObject_SetAttrString(p->helper_module, "pyside_feature_dict", p->feature_dict) < 0) + goto error; + + // This function will be disabled until phase 2 is done. + p->finish_import_func = nullptr; + + // Initialize the explicit init function. + AutoDecRef init(PyCFunction_NewEx(init_meth, nullptr, nullptr)); + if (init.isNull() + || PyDict_SetItemString(PyEval_GetBuiltins(), init_meth->ml_name, init) != 0) + goto error; + + return p; + } +error: + PyErr_Print(); + Py_FatalError("could not initialize part 1"); + return nullptr; +} + +static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) +{ + { + PyMethodDef *ml; + + // The single function to be called, but maybe more to come. + for (ml = methods; ml->ml_name != nullptr; ml++) { + PyObject *v = PyCFunction_NewEx(ml, nullptr, nullptr); + if (v == nullptr + || PyObject_SetAttrString(p->helper_module, ml->ml_name, v) != 0) + goto error; + Py_DECREF(v); + } + PyObject *bootstrap_func = PyObject_GetAttrString(p->helper_module, "bootstrap"); + if (bootstrap_func == nullptr) + goto error; + // The return value of the bootstrap function is the loader module. + PyObject *loader = PyObject_CallFunction(bootstrap_func, const_cast<char *>("()")); + if (loader == nullptr) + goto error; + // now the loader should be initialized + p->pyside_type_init_func = PyObject_GetAttrString(loader, "pyside_type_init"); + if (p->pyside_type_init_func == nullptr) + goto error; + p->create_signature_func = PyObject_GetAttrString(loader, "create_signature"); + if (p->create_signature_func == nullptr) + goto error; + p->seterror_argument_func = PyObject_GetAttrString(loader, "seterror_argument"); + if (p->seterror_argument_func == nullptr) + goto error; + p->make_helptext_func = PyObject_GetAttrString(loader, "make_helptext"); + if (p->make_helptext_func == nullptr) + goto error; + p->finish_import_func = PyObject_GetAttrString(loader, "finish_import"); + if (p->finish_import_func == nullptr) + goto error; + return 0; + } +error: + PyErr_Print(); + Py_FatalError("could not initialize part 2"); + return -1; +} + +#ifndef _WIN32 +//////////////////////////////////////////////////////////////////////////// +// a stack trace for linux-like platforms +#include <stdio.h> +#if defined(__GLIBC__) +# include <execinfo.h> +#endif +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> + +static void handler(int sig) { +#if defined(__GLIBC__) + void *array[30]; + size_t size; + + // get void *'s for all entries on the stack + size = backtrace(array, 30); + + // print out all the frames to stderr +#endif + fprintf(stderr, "Error: signal %d:\n", sig); +#if defined(__GLIBC__) + backtrace_symbols_fd(array, size, STDERR_FILENO); +#endif + exit(1); +} + +//////////////////////////////////////////////////////////////////////////// +#endif // _WIN32 + +safe_globals pyside_globals = nullptr; + +void init_module_1(void) +{ + static int init_done = 0; + + if (!init_done) { + pyside_globals = init_phase_1(init_methods); + if (pyside_globals != nullptr) + init_done = 1; + +#ifndef _WIN32 + // We enable the stack trace in CI, only. + const char *testEnv = getenv("QTEST_ENVIRONMENT"); + if (testEnv && strstr(testEnv, "ci")) + signal(SIGSEGV, handler); // install our handler +#endif // _WIN32 + + } +} + +void init_module_2(void) +{ + static int init_done = 0; + + if (!init_done) { + // Phase 2 will call __init__.py which touches a signature, itself. + // Therefore we set init_done prior to init_phase_2(). + init_done = 1; + init_phase_2(pyside_globals, signature_methods); + } +} + +} // extern "C" diff --git a/sources/shiboken6/libshiboken/signature/signature_helper.cpp b/sources/shiboken6/libshiboken/signature/signature_helper.cpp new file mode 100644 index 000000000..2b360c786 --- /dev/null +++ b/sources/shiboken6/libshiboken/signature/signature_helper.cpp @@ -0,0 +1,437 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +//////////////////////////////////////////////////////////////////////////// +// +// signature_helper.cpp +// -------------------- +// +// This file contains assoerted helper functions that are needed, +// but it is not helpful to see them all the time. +// + +#include "autodecref.h" +#include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" + +#include "signature_p.h" + +using namespace Shiboken; + +extern "C" { + +// Helper for __qualname__ which might not always exist in Python 2 (type). +PyObject *_get_qualname(PyObject *ob) +{ + // We support __qualname__ for types, only. + assert(PyType_Check(ob)); + PyObject *name = PyObject_GetAttr(ob, PyMagicName::qualname()); + if (name == nullptr) { + PyErr_Clear(); + name = PyObject_GetAttr(ob, PyMagicName::name()); + } + return name; +} + +static int _fixup_getset(PyTypeObject *type, const char *name, PyGetSetDef *new_gsp) +{ + /* + * This function pre-fills all fields of the new gsp. We then + * insert the changed values. + */ + PyGetSetDef *gsp = type->tp_getset; + if (gsp != nullptr) { + for (; gsp->name != nullptr; gsp++) { + if (strcmp(gsp->name, name) == 0) { + new_gsp->set = gsp->set; + new_gsp->doc = gsp->doc; + new_gsp->closure = gsp->closure; + return 1; // success + } + } + } + PyMemberDef *md = type->tp_members; + if (md != nullptr) + for (; md->name != nullptr; md++) + if (strcmp(md->name, name) == 0) + return 1; + // staticmethod has just a `__doc__` in the class + assert(strcmp(type->tp_name, "staticmethod") == 0 && strcmp(name, "__doc__") == 0); + return 0; +} + +int add_more_getsets(PyTypeObject *type, PyGetSetDef *gsp, PyObject **doc_descr) +{ + /* + * This function is used to assign a new `__signature__` attribute, + * and also to override a `__doc__` or `__name__` attribute. + */ + assert(PyType_Check(type)); + PyType_Ready(type); + PyObject *dict = type->tp_dict; + for (; gsp->name != nullptr; gsp++) { + PyObject *have_descr = PyDict_GetItemString(dict, gsp->name); + if (have_descr != nullptr) { + Py_INCREF(have_descr); + if (strcmp(gsp->name, "__doc__") == 0) + *doc_descr = have_descr; + else + assert(false); + if (!_fixup_getset(type, gsp->name, gsp)) + continue; + } + AutoDecRef descr(PyDescr_NewGetSet(type, gsp)); + if (descr.isNull()) + return -1; + if (PyDict_SetItemString(dict, gsp->name, descr) < 0) + return -1; + } + PyType_Modified(type); + return 0; +} + +static PyObject *get_funcname(PyObject *ob) +{ + PyObject *func = ob; + if (Py_TYPE(ob) == PepStaticMethod_TypePtr) + func = PyObject_GetAttr(ob, PyMagicName::func()); + else + Py_INCREF(func); + PyObject *func_name = PyObject_GetAttr(func, PyMagicName::name()); + Py_DECREF(func); + if (func_name == nullptr) + Py_FatalError("unexpected name problem in compute_name_key"); + return func_name; +} + +static PyObject *compute_name_key(PyObject *ob) +{ + if (PyType_Check(ob)) + return GetTypeKey(ob); + AutoDecRef func_name(get_funcname(ob)); + AutoDecRef type_key(GetTypeKey(GetClassOrModOf(ob))); + return Py_BuildValue("(OO)", type_key.object(), func_name.object()); +} + +static PyObject *_func_with_new_name(PyTypeObject *type, + PyMethodDef *meth, + const char *new_name) +{ + /* + * Create a function with a lower case name. + * Note: This is similar to feature_select's methodWithNewName, + * but does not create a descriptor. + * XXX Maybe we can get rid of this, completely? + */ + auto obtype = reinterpret_cast<PyObject *>(type); + int len = strlen(new_name); + auto name = new char[len + 1]; + strcpy(name, new_name); + auto new_meth = new PyMethodDef; + new_meth->ml_name = name; + new_meth->ml_meth = meth->ml_meth; + new_meth->ml_flags = meth->ml_flags; + new_meth->ml_doc = meth->ml_doc; + return PyCFunction_NewEx(new_meth, obtype, nullptr); +} + +static int build_name_key_to_func(PyObject *obtype) +{ + auto *type = reinterpret_cast<PyTypeObject *>(obtype); + PyMethodDef *meth = type->tp_methods; + + if (meth == nullptr) + return 0; + + AutoDecRef type_key(GetTypeKey(obtype)); + for (; meth->ml_name != nullptr; meth++) { + AutoDecRef func(PyCFunction_NewEx(meth, obtype, nullptr)); + AutoDecRef func_name(get_funcname(func)); + AutoDecRef name_key(Py_BuildValue("(OO)", type_key.object(), func_name.object())); + if (func.isNull() || name_key.isNull() + || PyDict_SetItem(pyside_globals->map_dict, name_key, func) < 0) + return -1; + } + // PYSIDE-1019: Now we repeat the same for snake case names. + meth = type->tp_methods; + for (; meth->ml_name != nullptr; meth++) { + const char *name = String::toCString(String::getSnakeCaseName(meth->ml_name, true)); + AutoDecRef func(_func_with_new_name(type, meth, name)); + AutoDecRef func_name(get_funcname(func)); + AutoDecRef name_key(Py_BuildValue("(OO)", type_key.object(), func_name.object())); + if (func.isNull() || name_key.isNull() + || PyDict_SetItem(pyside_globals->map_dict, name_key, func) < 0) + return -1; + } + return 0; +} + +PyObject *name_key_to_func(PyObject *ob) +{ + /* + * We build a mapping from name_key to function. + * This could also be computed directly, but the Limited API + * makes this impossible. So we always build our own mapping. + */ + AutoDecRef name_key(compute_name_key(ob)); + if (name_key.isNull()) + Py_RETURN_NONE; + + PyObject *ret = PyDict_GetItem(pyside_globals->map_dict, name_key); + if (ret == nullptr) { + // do a lazy initialization + AutoDecRef type_key(GetTypeKey(GetClassOrModOf(ob))); + PyObject *type = PyDict_GetItem(pyside_globals->map_dict, + type_key); + if (type == nullptr) + Py_RETURN_NONE; + assert(PyType_Check(type)); + if (build_name_key_to_func(type) < 0) + return nullptr; + ret = PyDict_GetItem(pyside_globals->map_dict, name_key); + } + Py_XINCREF(ret); + return ret; +} + +static PyObject *_build_new_entry(PyObject *new_name, PyObject *value) +{ + PyObject *new_value = PyDict_Copy(value); + PyObject *multi = PyDict_GetItem(value, PyName::multi()); + if (multi != nullptr && Py_TYPE(multi) == &PyList_Type) { + ssize_t len = PyList_Size(multi); + AutoDecRef list(PyList_New(len)); + if (list.isNull()) + return nullptr; + for (int idx = 0; idx < len; ++idx) { + auto multi_entry = PyList_GetItem(multi, idx); + auto dup = PyDict_Copy(multi_entry); + if (PyDict_SetItem(dup, PyName::name(), new_name) < 0) + return nullptr; + if (PyList_SetItem(list, idx, dup) < 0) + return nullptr; + } + if (PyDict_SetItem(new_value, PyName::multi(), list) < 0) + return nullptr; + } else { + if (PyDict_SetItem(new_value, PyName::name(), new_name) < 0) + return nullptr; + } + return new_value; +} + +int insert_snake_case_variants(PyObject *dict) +{ + AutoDecRef snake_dict(PyDict_New()); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + AutoDecRef name(String::getSnakeCaseName(key, true)); + AutoDecRef new_value(_build_new_entry(name, value)); + if (PyDict_SetItem(snake_dict, name, new_value) < 0) + return -1; + } + return PyDict_Merge(dict, snake_dict, 0); +} + +PyObject *_get_class_of_cf(PyObject *ob_cf) +{ + PyObject *selftype = PyCFunction_GET_SELF(ob_cf); + if (selftype == nullptr) { + selftype = PyDict_GetItem(pyside_globals->map_dict, ob_cf); + if (selftype == nullptr) { + // This must be an overloaded function that we handled special. + AutoDecRef special(Py_BuildValue("(OO)", ob_cf, PyName::overload())); + selftype = PyDict_GetItem(pyside_globals->map_dict, special); + if (selftype == nullptr) { + // This is probably a module function. We will return type(None). + selftype = Py_None; + } + } + } + + PyObject *obtype_mod = (PyType_Check(selftype) || PyModule_Check(selftype)) + ? selftype + : reinterpret_cast<PyObject *>(Py_TYPE(selftype)); + Py_INCREF(obtype_mod); + return obtype_mod; +} + +PyObject *_get_class_of_sm(PyObject *ob_sm) +{ + AutoDecRef func(PyObject_GetAttr(ob_sm, PyMagicName::func())); + return _get_class_of_cf(func); +} + +PyObject *_get_class_of_descr(PyObject *ob) +{ + return PyObject_GetAttr(ob, PyMagicName::objclass()); +} + +PyObject *_address_to_stringlist(PyObject *numkey) +{ + /* + * This is a tiny optimization that saves initialization time. + * Instead of creating all Python strings during the call to + * `PySide_BuildSignatureArgs`, we store the address of the stringlist. + * When needed in `PySide_BuildSignatureProps`, the strings are + * finally materialized. + */ + ssize_t address = PyNumber_AsSsize_t(numkey, PyExc_ValueError); + if (address == -1 && PyErr_Occurred()) + return nullptr; + char **sig_strings = reinterpret_cast<char **>(address); + PyObject *res_list = PyList_New(0); + if (res_list == nullptr) + return nullptr; + for (; *sig_strings != nullptr; ++sig_strings) { + char *sig_str = *sig_strings; + AutoDecRef pystr(Py_BuildValue("s", sig_str)); + if (pystr.isNull() || PyList_Append(res_list, pystr) < 0) + return nullptr; + } + return res_list; +} + +static int _build_func_to_type(PyObject *obtype) +{ + /* + * There is no general way to directly get the type of a static method. + * On Python 3, the type is hidden in an unused pointer in the + * PyCFunction structure, but the Limited API does not allow to access + * this, either. + * + * In the end, it was easier to avoid such tricks and build an explicit + * mapping from function to type. + * + * We walk through the method list of the type + * and record the mapping from static method to this type in a dict. + * We also check for hidden methods, see below. + */ + auto *type = reinterpret_cast<PyTypeObject *>(obtype); + PyObject *dict = type->tp_dict; + PyMethodDef *meth = type->tp_methods; + + if (meth == nullptr) + return 0; + + for (; meth->ml_name != nullptr; meth++) { + /* + * It is possible that a method is overwritten by another + * attribute with the same name. This case was obviously provoked + * explicitly in "testbinding.TestObject.staticMethodDouble", + * where instead of the method a "PySide2.QtCore.Signal" object + * was in the dict. + * This overlap is also found in regular PySide under + * "PySide2.QtCore.QProcess.error" where again a signal object is + * returned. These hidden methods will be opened for the + * signature module by adding them under the name + * "{name}.overload". + */ + PyObject *descr = PyDict_GetItemString(dict, meth->ml_name); + PyObject *look_attr = meth->ml_flags & METH_STATIC ? PyMagicName::func() + : PyMagicName::name(); + int check_name = meth->ml_flags & METH_STATIC ? 0 : 1; + if (descr == nullptr) + return -1; + + // We first check all methods if one is hidden by something else. + AutoDecRef look(PyObject_GetAttr(descr, look_attr)); + AutoDecRef given(Py_BuildValue("s", meth->ml_name)); + if (look.isNull() + || (check_name && PyObject_RichCompareBool(look, given, Py_EQ) != 1)) { + PyErr_Clear(); + AutoDecRef cfunc(PyCFunction_NewEx( + meth, reinterpret_cast<PyObject *>(type), nullptr)); + if (cfunc.isNull()) + return -1; + if (meth->ml_flags & METH_STATIC) + descr = PyStaticMethod_New(cfunc); + else + descr = PyDescr_NewMethod(type, meth); + if (descr == nullptr) + return -1; + char mangled_name[200]; + strcpy(mangled_name, meth->ml_name); + strcat(mangled_name, ".overload"); + if (PyDict_SetItemString(dict, mangled_name, descr) < 0) + return -1; + if (meth->ml_flags & METH_STATIC) { + // This is the special case where a static method is hidden. + AutoDecRef special(Py_BuildValue("(Os)", cfunc.object(), "overload")); + if (PyDict_SetItem(pyside_globals->map_dict, special, obtype) < 0) + return -1; + } + if (PyDict_SetItemString(pyside_globals->map_dict, mangled_name, obtype) < 0) + return -1; + continue; + } + // Then we insert the mapping for static methods. + if (meth->ml_flags & METH_STATIC) { + if (PyDict_SetItem(pyside_globals->map_dict, look, obtype) < 0) + return -1; + } + } + return 0; +} + +int _finish_nested_classes(PyObject *obdict) +{ + PyObject *key, *value, *obtype; + PyTypeObject *subtype; + Py_ssize_t pos = 0; + + if (obdict == nullptr) + return -1; + while (PyDict_Next(obdict, &pos, &key, &value)) { + if (PyType_Check(value)) { + obtype = value; + if (_build_func_to_type(obtype) < 0) + return -1; + // now continue with nested cases + subtype = reinterpret_cast<PyTypeObject *>(obtype); + if (_finish_nested_classes(subtype->tp_dict) < 0) + return -1; + } + } + return 0; +} + +} // extern "C" diff --git a/sources/shiboken6/libshiboken/signature/signature_p.h b/sources/shiboken6/libshiboken/signature/signature_p.h new file mode 100644 index 000000000..ef7846472 --- /dev/null +++ b/sources/shiboken6/libshiboken/signature/signature_p.h @@ -0,0 +1,107 @@ +/**************************************************************************** +** +** Copyright (C) 2020 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef SIGNATURE_IMPL_H +#define SIGNATURE_IMPL_H + +#include "signature.h" + +extern "C" { + +// signature_globals.cpp + +typedef struct safe_globals_struc { + // init part 1: get arg_dict + PyObject *helper_module; + PyObject *arg_dict; + PyObject *map_dict; + PyObject *value_dict; // for writing signatures + PyObject *feature_dict; // registry for PySide.support.__feature__ + // init part 2: run module + PyObject *pyside_type_init_func; + PyObject *create_signature_func; + PyObject *seterror_argument_func; + PyObject *make_helptext_func; + PyObject *finish_import_func; +} safe_globals_struc, *safe_globals; + +extern safe_globals pyside_globals; +extern PyMethodDef signature_methods[]; + +void init_module_1(void); +void init_module_2(void); + +// signature.cpp + +PyObject *GetTypeKey(PyObject *ob); + +PyObject *GetSignature_Function(PyObject *, PyObject *); +PyObject *GetSignature_TypeMod(PyObject *, PyObject *); +PyObject *GetSignature_Wrapper(PyObject *, PyObject *); + +PyObject *get_signature_intern(PyObject *ob, PyObject *modifier); +PyObject *PySide_BuildSignatureProps(PyObject *class_mod); +PyObject *GetClassOrModOf(PyObject *ob); + +// signature_extend.cpp + +PyObject *pyside_cf_get___signature__(PyObject *func, PyObject *modifier); +PyObject *pyside_sm_get___signature__(PyObject *sm, PyObject *modifier); +PyObject *pyside_md_get___signature__(PyObject *ob_md, PyObject *modifier); +PyObject *pyside_wd_get___signature__(PyObject *ob, PyObject *modifier); +PyObject *pyside_tp_get___signature__(PyObject *obtype_mod, PyObject *modifier); + +int PySide_PatchTypes(void); +PyObject *pyside_tp_get___doc__(PyObject *tp); + +// signature_helper.cpp + +PyObject *_get_qualname(PyObject *ob); +int add_more_getsets(PyTypeObject *type, PyGetSetDef *gsp, PyObject **doc_descr); +PyObject *name_key_to_func(PyObject *ob); +int insert_snake_case_variants(PyObject *dict); +PyObject *_get_class_of_cf(PyObject *ob_cf); +PyObject *_get_class_of_sm(PyObject *ob_sm); +PyObject *_get_class_of_descr(PyObject *ob); +PyObject *_address_to_stringlist(PyObject *numkey); +int _finish_nested_classes(PyObject *dict); + +} // extern "C" + +#endif // SIGNATURE_IMPL_H |