aboutsummaryrefslogtreecommitdiffstats
path: root/sources/shiboken2
diff options
context:
space:
mode:
authorChristian Tismer <tismer@stackless.com>2019-04-12 10:58:02 +0200
committerChristian Tismer <tismer@stackless.com>2019-04-16 08:07:38 +0000
commitc9f522f082485186dcd8be80186e3c11b55f2b77 (patch)
tree24a95c682a4ba6ce712c229228c7b74e733b67f8 /sources/shiboken2
parente0aa898c068006a7b6aef4cdb0528f2d0b8e0a94 (diff)
Ensure that signature strings never overflow again
The signature module used to use large strings with the signatures of all functions in a class. This can lead to an overflow in MSVC, because the maximum string length funnily still is 32K unicode characters. This patch solves that by using a single string per function. Instead of a huge string, a list of strings is passed to each class. To prevent any runtime increase, the string list creation is deferred until the actual usage. At initialization time only a ssize_t holding the structure address is passed. As a result, the signature module should be even slightly faster. Task-number: PYSIDE-955 Change-Id: I99faf942a3cca03456928b8aec5e8a4b9924b8b2 Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
Diffstat (limited to 'sources/shiboken2')
-rw-r--r--sources/shiboken2/generator/shiboken2/cppgenerator.cpp20
-rw-r--r--sources/shiboken2/libshiboken/basewrapper.cpp4
-rw-r--r--sources/shiboken2/libshiboken/basewrapper.h2
-rw-r--r--sources/shiboken2/libshiboken/signature.cpp58
-rw-r--r--sources/shiboken2/libshiboken/signature.h4
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py14
6 files changed, 63 insertions, 39 deletions
diff --git a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
index b6d3687ac..29220c739 100644
--- a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
+++ b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
@@ -664,7 +664,7 @@ void CppGenerator::generateClass(QTextStream &s, GeneratorContext &classContext)
s << NULL_PTR;
s << "}," << endl;
}
- s << INDENT << '{' << NULL_PTR << "} // Sentinel" << endl;
+ s << INDENT << '{' << NULL_PTR << "} // Sentinel" << endl;
s << "};" << endl << endl;
}
@@ -4924,11 +4924,11 @@ void CppGenerator::writeClassRegister(QTextStream &s,
// PYSIDE-510: Create a signatures string for the introspection feature.
s << "// The signatures string for the functions." << endl;
s << "// Multiple signatures have their index \"n:\" in front." << endl;
- s << "const char " << initFunctionName << "_SignaturesString[] = \"\"" << endl;
+ s << "static const char *" << initFunctionName << "_SignatureStrings[] = {" << endl;
QString line;
while (signatureStream.readLineInto(&line))
- s << INDENT << '"' << line << "\\n\"" << endl;
- s << ';' << endl << endl;
+ s << INDENT << '"' << line << "\"," << endl;
+ s << INDENT << NULL_PTR << "}; // Sentinel" << endl << endl;
s << "void init_" << initFunctionName;
s << "(PyObject* " << enclosingObjectVariable << ")" << endl;
s << '{' << endl;
@@ -4981,8 +4981,8 @@ void CppGenerator::writeClassRegister(QTextStream &s,
// 4:typeSpec
s << INDENT << '&' << chopType(pyTypeName) << "_spec," << endl;
- // 5:signaturesString
- s << INDENT << initFunctionName << "_SignaturesString," << endl;
+ // 5:signatureStrings
+ s << INDENT << initFunctionName << "_SignatureStrings," << endl;
// 6:cppObjDtor
s << INDENT;
@@ -5661,11 +5661,11 @@ bool CppGenerator::finishGeneration()
// PYSIDE-510: Create a signatures string for the introspection feature.
s << "// The signatures string for the global functions." << endl;
s << "// Multiple signatures have their index \"n:\" in front." << endl;
- s << "const char " << moduleName() << "_SignaturesString[] = \"\"" << endl;
+ s << "static const char *" << moduleName() << "_SignatureStrings[] = {" << endl;
QString line;
while (signatureStream.readLineInto(&line))
- s << INDENT << '"' << line << "\\n\"" << endl;
- s << INDENT << ';' << endl << endl;
+ s << INDENT << '"' << line << "\"," << endl;
+ s << INDENT << NULL_PTR << "}; // Sentinel" << endl << endl;
s << "SBK_MODULE_INIT_FUNCTION_BEGIN(" << moduleName() << ")" << endl;
@@ -5799,7 +5799,7 @@ bool CppGenerator::finishGeneration()
// finish the rest of __signature__ initialization.
s << INDENT << "FinishSignatureInitialization(module, " << moduleName()
- << "_SignaturesString);" << endl;
+ << "_SignatureStrings);" << endl;
if (usePySideExtensions()) {
// initialize the qApp module.
diff --git a/sources/shiboken2/libshiboken/basewrapper.cpp b/sources/shiboken2/libshiboken/basewrapper.cpp
index db65ec696..3a043d849 100644
--- a/sources/shiboken2/libshiboken/basewrapper.cpp
+++ b/sources/shiboken2/libshiboken/basewrapper.cpp
@@ -744,7 +744,7 @@ introduceWrapperType(PyObject *enclosingObject,
const char *typeName,
const char *originalName,
PyType_Spec *typeSpec,
- const char *signaturesString,
+ const char *signatureStrings[],
ObjectDestructor cppObjDtor,
SbkObjectType *baseType,
PyObject *baseTypes,
@@ -765,7 +765,7 @@ introduceWrapperType(PyObject *enclosingObject,
}
}
// PYSIDE-510: Here is the single change to support signatures.
- if (SbkSpecial_Type_Ready(enclosingObject, reinterpret_cast<PyTypeObject *>(type), signaturesString) < 0)
+ if (SbkSpecial_Type_Ready(enclosingObject, reinterpret_cast<PyTypeObject *>(type), signatureStrings) < 0)
return nullptr;
initPrivateData(type);
diff --git a/sources/shiboken2/libshiboken/basewrapper.h b/sources/shiboken2/libshiboken/basewrapper.h
index e1cc64ba9..d12b7222b 100644
--- a/sources/shiboken2/libshiboken/basewrapper.h
+++ b/sources/shiboken2/libshiboken/basewrapper.h
@@ -219,7 +219,7 @@ LIBSHIBOKEN_API SbkObjectType *introduceWrapperType(PyObject *enclosingObject,
const char *typeName,
const char *originalName,
PyType_Spec *typeSpec,
- const char *signaturesString,
+ const char *signatureStrings[],
ObjectDestructor cppObjDtor,
SbkObjectType *baseType,
PyObject *baseTypes,
diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp
index 8d580b487..3defca7d2 100644
--- a/sources/shiboken2/libshiboken/signature.cpp
+++ b/sources/shiboken2/libshiboken/signature.cpp
@@ -61,6 +61,7 @@ extern "C"
#define PYTHON_NEEDS_ITERATOR_FLAG (!PYTHON_IS_PYTHON3)
#define PYTHON_EXPOSES_METHODDESCR (PYTHON_IS_PYTHON3)
#define PYTHON_NO_TYPE_IN_FUNCTIONS (!PYTHON_IS_PYTHON3 || Py_LIMITED_API)
+#define PYTHON_HAS_INT_AND_LONG (!PYTHON_IS_PYTHON3)
// These constants are still in use:
#define PYTHON_USES_D_COMMON (PY_VERSION_HEX >= 0x03020000)
@@ -326,7 +327,7 @@ TypeKey_to_PropsDict(PyObject *type_key, PyObject *obtype)
dict = empty_dict;
}
if (!PyDict_Check(dict))
- dict = PySide_BuildSignatureProps(obtype);
+ dict = PySide_BuildSignatureProps(type_key);
return dict;
}
@@ -821,16 +822,23 @@ init_module_1(void)
}
static int
-PySide_BuildSignatureArgs(PyObject *obtype_mod, const char *signatures)
+PySide_BuildSignatureArgs(PyObject *obtype_mod, const char *signatures[])
{
init_module_1();
Shiboken::AutoDecRef type_key(GetTypeKey(obtype_mod));
- Shiboken::AutoDecRef arg_tup(Py_BuildValue("(Os)", obtype_mod, signatures));
- if (type_key.isNull() || arg_tup.isNull()
- || PyDict_SetItem(pyside_globals->arg_dict, type_key, arg_tup) < 0)
+ /*
+ * PYSIDE-996: Avoid string overflow in MSVC, which has a limit of
+ * 2**15 unicode characters (64 K memory).
+ * Instead of one huge string, we take a ssize_t that is the
+ * address of a string array. It will not be turned into a real
+ * string list until really used by Python. This is quite optimal.
+ */
+ Shiboken::AutoDecRef numkey(Py_BuildValue("n", signatures));
+ if (type_key.isNull() || numkey.isNull()
+ || PyDict_SetItem(pyside_globals->arg_dict, type_key, numkey) < 0)
return -1;
/*
- * We also record a mapping from type key to type/module. This helps to
+ * We record also a mapping from type key to type/module. This helps to
* lazily initialize the Py_LIMITED_API in name_key_to_func().
*/
return PyDict_SetItem(pyside_globals->map_dict, type_key, obtype_mod) == 0 ? 0 : -1;
@@ -856,7 +864,26 @@ init_module_2(void)
}
static PyObject *
-PySide_BuildSignatureProps(PyObject *obtype_mod)
+_address_to_stringlist(PyObject *numkey)
+{
+ ssize_t address = PyNumber_AsSsize_t(numkey, PyExc_ValueError);
+ if (address == -1 && PyErr_Occurred())
+ return nullptr;
+ char **sig_strings = reinterpret_cast<char **>(address);
+ PyObject *res_list = PyList_New(0);
+ if (res_list == nullptr)
+ return nullptr;
+ for (; *sig_strings != nullptr; ++sig_strings) {
+ char *sig_str = *sig_strings;
+ Shiboken::AutoDecRef pystr(Py_BuildValue("s", sig_str));
+ if (pystr.isNull() || PyList_Append(res_list, pystr) < 0)
+ return nullptr;
+ }
+ return res_list;
+}
+
+static PyObject *
+PySide_BuildSignatureProps(PyObject *type_key)
{
/*
* Here is the second part of the function.
@@ -865,11 +892,14 @@ PySide_BuildSignatureProps(PyObject *obtype_mod)
* them by the function result.
*/
init_module_2();
- Shiboken::AutoDecRef type_key(GetTypeKey(obtype_mod));
- if (type_key.isNull())
+ if (type_key == nullptr)
+ return nullptr;
+ PyObject *numkey = PyDict_GetItem(pyside_globals->arg_dict, type_key);
+ Shiboken::AutoDecRef strings(_address_to_stringlist(numkey));
+ if (strings.isNull())
return nullptr;
- PyObject *arg_tup = PyDict_GetItem(pyside_globals->arg_dict, type_key);
- if (arg_tup == nullptr)
+ Shiboken::AutoDecRef arg_tup(Py_BuildValue("(OO)", type_key, strings.object()));
+ if (arg_tup.isNull())
return nullptr;
PyObject *dict = PyObject_CallObject(pyside_globals->pyside_type_init_func, arg_tup);
if (dict == nullptr) {
@@ -890,7 +920,7 @@ static int _finish_nested_classes(PyObject *dict);
static int _build_func_to_type(PyObject *obtype);
static int
-PySide_FinishSignatures(PyObject *module, const char *signatures)
+PySide_FinishSignatures(PyObject *module, const char *signatures[])
{
/*
* Initialization of module functions and resolving of static methods.
@@ -1030,7 +1060,7 @@ _build_func_to_type(PyObject *obtype)
int
SbkSpecial_Type_Ready(PyObject *module, PyTypeObject *type,
- const char *signatures)
+ const char *signatures[])
{
if (PyType_Ready(type) < 0)
return -1;
@@ -1044,7 +1074,7 @@ SbkSpecial_Type_Ready(PyObject *module, PyTypeObject *type,
}
void
-FinishSignatureInitialization(PyObject *module, const char *signatures)
+FinishSignatureInitialization(PyObject *module, const char *signatures[])
{
/*
* This function is called at the very end of a module initialization.
diff --git a/sources/shiboken2/libshiboken/signature.h b/sources/shiboken2/libshiboken/signature.h
index a36b03d4f..6b477c52e 100644
--- a/sources/shiboken2/libshiboken/signature.h
+++ b/sources/shiboken2/libshiboken/signature.h
@@ -45,8 +45,8 @@
extern "C"
{
-LIBSHIBOKEN_API int SbkSpecial_Type_Ready(PyObject *, PyTypeObject *, const char *);
-LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char *);
+LIBSHIBOKEN_API int SbkSpecial_Type_Ready(PyObject *, PyTypeObject *, const char *[]);
+LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char *[]);
LIBSHIBOKEN_API void SetError_Argument(PyObject *, const char *);
} // extern "C"
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
index 09e78f1b0..72ca35757 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
@@ -224,7 +224,6 @@ def _resolve_type(thing, line):
return _resolve_value(thing, None, line)
def calculate_props(line):
- line = line.strip()
res = _parse_line(line)
arglist = res["arglist"]
annotations = {}
@@ -257,8 +256,7 @@ def calculate_props(line):
props["multi"] = res["multi"]
return props
-def fixup_multilines(sig_str):
- lines = list(line.strip() for line in sig_str.strip().splitlines())
+def fixup_multilines(lines):
res = []
multi_lines = []
for line in lines:
@@ -282,15 +280,11 @@ def fixup_multilines(sig_str):
res.append(line)
return res
-def pyside_type_init(typemod, sig_str):
+def pyside_type_init(type_key, sig_strings):
dprint()
- if type(typemod) is types.ModuleType:
- dprint("Initialization of module '{}'".format(typemod.__name__))
- else:
- dprint("Initialization of type '{}.{}'".format(typemod.__module__,
- typemod.__name__))
+ dprint("Initialization of type key '{}'".format(type_key))
update_mapping()
- lines = fixup_multilines(sig_str)
+ lines = fixup_multilines(sig_strings)
ret = {}
multi_props = []
for line in lines: