aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristian Tismer <tismer@stackless.com>2024-02-12 16:24:37 +0100
committerChristian Tismer <tismer@stackless.com>2024-03-13 14:53:07 +0100
commit7accf7c3042e3f0680fa0615a0f13b54d28a0efd (patch)
treee9f0c87d5bcb67a74a5a60be8374277eb48f545c
parenta6ebf276fd654dd63becdab24d9a9b6a5594f076 (diff)
LazyInit: Implement Lazy Initialization by Delayed Module Entries
Lazy init is done by module entries which are delayed. Although visible in the module, the classes are only created when actually accessed by getattr. Internally, the access to the global Init_xxx functions is redirected to a Shiboken::Module::get function which resolves the classes if not already present in the global type array. PYSIDE6_OPTION_LAZY 0 - no lazy loading 1 - lazy load all known modules 2 - lazy load all modules Task-number: PYSIDE-2404 Change-Id: I98c01856e293732c166662050d0fbc6f6ec9082b Reviewed-by: Shyamnath Premnadh <Shyamnath.Premnadh@qt.io> Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
-rw-r--r--sources/pyside6/PySide6/glue/qtcore.cpp2
-rw-r--r--sources/pyside6/libpyside/pyside.cpp8
-rw-r--r--sources/shiboken6/generator/shiboken/cppgenerator.cpp72
-rw-r--r--sources/shiboken6/generator/shiboken/cppgenerator.h3
-rw-r--r--sources/shiboken6/generator/shiboken/headergenerator.cpp1
-rw-r--r--sources/shiboken6/generator/shiboken/shibokengenerator.cpp30
-rw-r--r--sources/shiboken6/generator/shiboken/shibokengenerator.h4
-rw-r--r--sources/shiboken6/libshiboken/basewrapper.cpp1
-rw-r--r--sources/shiboken6/libshiboken/sbkconverter.cpp27
-rw-r--r--sources/shiboken6/libshiboken/sbkcppstring.cpp5
-rw-r--r--sources/shiboken6/libshiboken/sbkcppstring.h2
-rw-r--r--sources/shiboken6/libshiboken/sbkmodule.cpp423
-rw-r--r--sources/shiboken6/libshiboken/sbkmodule.h33
-rw-r--r--sources/shiboken6/libshiboken/signature/signature.cpp2
14 files changed, 574 insertions, 39 deletions
diff --git a/sources/pyside6/PySide6/glue/qtcore.cpp b/sources/pyside6/PySide6/glue/qtcore.cpp
index 8438df976..e48df2679 100644
--- a/sources/pyside6/PySide6/glue/qtcore.cpp
+++ b/sources/pyside6/PySide6/glue/qtcore.cpp
@@ -671,6 +671,7 @@ return %CONVERTTOPYTHON[QByteArray](ba);
// @snippet qbytearray-mgetitem
// @snippet qbytearray-msetitem
+// PYSIDE-2404: Usage of the `get()` function not necessary, the type exists.
if (PyIndex_Check(_key)) {
Py_ssize_t _i = PyNumber_AsSsize_t(_key, PyExc_IndexError);
if (_i == -1 && PyErr_Occurred())
@@ -1806,6 +1807,7 @@ if (dataChar == nullptr) {
// @snippet qdatastream-read-bytes
// @snippet qloggingcategory_to_cpp
+// PYSIDE-2404: Usage of the `get()` function not necessary, the type exists.
QLoggingCategory *category{nullptr};
Shiboken::Conversions::pythonToCppPointer(SbkPySide6_QtCoreTypes[SBK_QLoggingCategory_IDX],
pyArgs[0], &(category));
diff --git a/sources/pyside6/libpyside/pyside.cpp b/sources/pyside6/libpyside/pyside.cpp
index 074c289b8..f4f113d34 100644
--- a/sources/pyside6/libpyside/pyside.cpp
+++ b/sources/pyside6/libpyside/pyside.cpp
@@ -32,6 +32,7 @@
#include <sbkstring.h>
#include <sbkstaticstrings.h>
#include <sbkfeature_base.h>
+#include <sbkmodule.h>
#include <QtCore/QByteArray>
#include <QtCore/QCoreApplication>
@@ -687,6 +688,13 @@ static const char *typeName(const QObject *cppSelf)
typeName = name;
break;
}
+ // PYSIDE-2404: Did not find the name. Load the lazy classes
+ // which have this name and try again.
+ Shiboken::Module::loadLazyClassesWithName(name);
+ if (Shiboken::Conversions::getConverter(name)) {
+ typeName = name;
+ break;
+ }
}
}
return typeName;
diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.cpp b/sources/shiboken6/generator/shiboken/cppgenerator.cpp
index 9bd6af7c1..55da21d8b 100644
--- a/sources/shiboken6/generator/shiboken/cppgenerator.cpp
+++ b/sources/shiboken6/generator/shiboken/cppgenerator.cpp
@@ -4182,6 +4182,8 @@ void CppGenerator::writeExtendedConverterInitialization(TextStream &s,
QString targetTypeName = fixedCppTypeName(externalType);
QString toCpp = pythonToCppFunctionName(sourceTypeName, targetTypeName);
QString isConv = convertibleToCppFunctionName(sourceTypeName, targetTypeName);
+ if (!externalType->isPrimitive())
+ s << cpythonTypeNameExt(externalType) << ";\n";
writeAddPythonToCppConversion(s, converterVar, toCpp, isConv);
}
}
@@ -5236,7 +5238,7 @@ bool CppGenerator::writeEnumInitialization(TextStream &s, const AbstractMetaEnum
bool etypeUsed = false;
- QString enumVarTypeObj = cpythonTypeNameExt(enumTypeEntry);
+ QString enumVarTypeObj = cpythonTypeNameExtSet(enumTypeEntry);
if (!cppEnum.isAnonymous()) {
int packageLevel = packageName().count(u'.') + 1;
s << "EType = Shiboken::Enum::"
@@ -5250,7 +5252,7 @@ bool CppGenerator::writeEnumInitialization(TextStream &s, const AbstractMetaEnum
if (cppEnum.typeEntry()->flags()) {
s << "// PYSIDE-1735: Mapping the flags class to the same enum class.\n"
- << cpythonTypeNameExt(cppEnum.typeEntry()->flags()) << " =\n"
+ << cpythonTypeNameExtSet(cppEnum.typeEntry()->flags()) << " =\n"
<< indent << "EType;\n" << outdent;
}
writeEnumConverterInitialization(s, cppEnum);
@@ -5365,8 +5367,8 @@ void CppGenerator::writeClassRegister(TextStream &s,
// PYSIDE-510: Create a signatures string for the introspection feature.
writeSignatureStrings(s, signatures, initFunctionName, "functions");
- s << "void init_" << initFunctionName;
- s << "(PyObject *" << enclosingObjectVariable << ")\n{\n" << indent;
+ s << "PyTypeObject *init_" << initFunctionName
+ << "(PyObject *" << enclosingObjectVariable << ")\n{\n" << indent;
// Multiple inheritance
QString pyTypeBasesVariable = chopType(pyTypeName) + u"_Type_bases"_s;
@@ -5459,9 +5461,9 @@ void CppGenerator::writeClassRegister(TextStream &s,
<< chopType(pyTypeName) << "_PropertyStrings);\n";
if (!classContext.forSmartPointer())
- s << cpythonTypeNameExt(classTypeEntry) << " = pyType;\n\n";
+ s << cpythonTypeNameExtSet(classTypeEntry) << " = pyType;\n\n";
else
- s << cpythonTypeNameExt(classContext.preciseType()) << " = pyType;\n\n";
+ s << cpythonTypeNameExtSet(classContext.preciseType()) << " = pyType;\n\n";
// Register conversions for the type.
writeConverterRegister(s, metaClass, classContext);
@@ -5534,15 +5536,31 @@ void CppGenerator::writeClassRegister(TextStream &s,
<< "));\n";
}
- s << outdent << "}\n";
+ s << "\nreturn pyType;\n" << outdent << "}\n";
}
void CppGenerator::writeStaticFieldInitialization(TextStream &s,
const AbstractMetaClassCPtr &metaClass)
{
- s << "\nvoid " << getSimpleClassStaticFieldsInitFunctionName(metaClass)
- << "()\n{\n" << indent << "Shiboken::AutoDecRef dict(PepType_GetDict(reinterpret_cast<PyTypeObject *>("
- << cpythonTypeName(metaClass) << ")));\n";
+ // cpythonTypeName == "Sbk_QRhiShaderResourceBinding_Data_TypeF"
+ QString name = cpythonTypeName(metaClass);
+ const auto parts = QStringView{name}.split(u'_', Qt::SkipEmptyParts);
+ if (parts.size() < 4) {
+ s << "\nPyTypeObject *" << getSimpleClassStaticFieldsInitFunctionName(metaClass)
+ << "(PyObject *module)\n{\n" << indent
+ << "auto *obType = PyObject_GetAttrString(module, \"" << metaClass->name() << "\");\n"
+ << "auto *type = reinterpret_cast<PyTypeObject *>(obType);\n"
+ << "Shiboken::AutoDecRef dict(PepType_GetDict(type));\n";
+ } else {
+ s << "\nPyTypeObject *" << getSimpleClassStaticFieldsInitFunctionName(metaClass)
+ << "(PyObject *module)\n{\n" << indent
+ << "auto *obContainerType = PyObject_GetAttrString(module, \""
+ << parts.at(1) << "\");\n"
+ << "auto *obType = PyObject_GetAttrString(obContainerType, \""
+ << parts.at(2) << "\");\n"
+ << "auto *type = reinterpret_cast<PyTypeObject *>(obType);\n"
+ << "Shiboken::AutoDecRef dict(PepType_GetDict(type));\n";
+ }
for (const AbstractMetaField &field : metaClass->fields()) {
if (field.isStatic()) {
s << "PyDict_SetItemString(dict, \"" << field.name()
@@ -5551,7 +5569,7 @@ void CppGenerator::writeStaticFieldInitialization(TextStream &s,
s << ");\n";
}
}
- s << '\n' << outdent << "}\n";
+ s << "return type;\n" << outdent << "}\n";
}
enum class QtRegisterMetaType
@@ -5873,18 +5891,28 @@ void CppGenerator::writeNbBoolFunction(const GeneratorContext &context,
// function.
void CppGenerator::writeInitFunc(TextStream &declStr, TextStream &callStr,
const QString &initFunctionName,
- const TypeEntryCPtr &enclosingEntry)
+ const TypeEntryCPtr &enclosingEntry,
+ const QString &pythonName)
{
const bool hasParent =
enclosingEntry && enclosingEntry->type() != TypeEntry::TypeSystemType;
- declStr << "void init_" << initFunctionName << "(PyObject *"
+ declStr << "PyTypeObject *init_" << initFunctionName << "(PyObject *"
<< (hasParent ? "enclosingClass" : "module") << ");\n";
- callStr << "init_" << initFunctionName;
if (hasParent) {
- callStr << "(reinterpret_cast<PyObject *>("
- << cpythonTypeNameExt(enclosingEntry) << "));\n";
+ const QString &enclosingName = enclosingEntry->name();
+ const auto parts = QStringView{enclosingName}.split(u"::", Qt::SkipEmptyParts);
+ callStr << "Shiboken::Module::AddTypeCreationFunction("
+ << "module, \"" << pythonName << "\", " << "init_" << initFunctionName << ", \"";
+ for (qsizetype i = 0; i < parts.size(); ++i) {
+ if (i > 0)
+ callStr << "\", \"";
+ callStr << parts.at(i);
+ }
+ callStr << "\");\n";
} else {
- callStr << "(module);\n";
+ callStr << "Shiboken::Module::AddTypeCreationFunction("
+ << "module, \"" << pythonName << "\", "
+ << "init_" << initFunctionName << ");\n";
}
}
@@ -5943,10 +5971,10 @@ bool CppGenerator::finishGeneration()
}
writeInitFunc(s_classInitDecl, s_classPythonDefines,
getSimpleClassInitFunctionName(cls),
- targetLangEnclosingEntry(te));
+ targetLangEnclosingEntry(te), cls->name());
if (cls->hasStaticFields()) {
- s_classInitDecl << "void "
- << getSimpleClassStaticFieldsInitFunctionName(cls) << "();\n";
+ s_classInitDecl << "PyTypeObject *"
+ << getSimpleClassStaticFieldsInitFunctionName(cls) << "(PyObject *module);\n";
classesWithStaticFields.append(cls);
}
if (hasConfigCondition) {
@@ -5966,7 +5994,7 @@ bool CppGenerator::finishGeneration()
writeInitFunc(s_classInitDecl, s_classPythonDefines,
getInitFunctionName(context),
- enclosingTypeEntry);
+ enclosingTypeEntry, smp.type.name());
includes.insert(smp.type.instantiations().constFirst().typeEntry()->include());
}
@@ -6304,7 +6332,7 @@ bool CppGenerator::finishGeneration()
s << "\n// Static field initialization\n";
for (const auto &cls : std::as_const(classesWithStaticFields)) {
ConfigurableScope configScope(s, cls->typeEntry());
- s << getSimpleClassStaticFieldsInitFunctionName(cls) << "();\n";
+ s << getSimpleClassStaticFieldsInitFunctionName(cls) << "(module);\n";
}
}
diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.h b/sources/shiboken6/generator/shiboken/cppgenerator.h
index 95ed0bf7d..b7b1590ae 100644
--- a/sources/shiboken6/generator/shiboken/cppgenerator.h
+++ b/sources/shiboken6/generator/shiboken/cppgenerator.h
@@ -55,7 +55,8 @@ private:
const AbstractMetaClassCList &innerClasses = {}) const;
static void writeInitFunc(TextStream &declStr, TextStream &callStr,
const QString &initFunctionName,
- const TypeEntryCPtr &enclosingEntry = {});
+ const TypeEntryCPtr &enclosingEntry,
+ const QString &pythonName);
static void writeCacheResetNative(TextStream &s, const GeneratorContext &classContext);
void writeConstructorNative(TextStream &s, const GeneratorContext &classContext,
const AbstractMetaFunctionCPtr &func) const;
diff --git a/sources/shiboken6/generator/shiboken/headergenerator.cpp b/sources/shiboken6/generator/shiboken/headergenerator.cpp
index a07d3d2d5..12fff574c 100644
--- a/sources/shiboken6/generator/shiboken/headergenerator.cpp
+++ b/sources/shiboken6/generator/shiboken/headergenerator.cpp
@@ -791,6 +791,7 @@ bool HeaderGenerator::finishGeneration()
}
s << "#include <sbkpython.h>\n";
+ s << "#include <sbkmodule.h>\n";
s << "#include <sbkconverter.h>\n";
QStringList requiredTargetImports = TypeDatabase::instance()->requiredTargetImports();
diff --git a/sources/shiboken6/generator/shiboken/shibokengenerator.cpp b/sources/shiboken6/generator/shiboken/shibokengenerator.cpp
index 71e13f48a..052292329 100644
--- a/sources/shiboken6/generator/shiboken/shibokengenerator.cpp
+++ b/sources/shiboken6/generator/shiboken/shibokengenerator.cpp
@@ -335,7 +335,7 @@ QString ShibokenGenerator::fullPythonClassName(const AbstractMetaClassCPtr &meta
fullClassName.prepend(enclosing->name() + u'.');
enclosing = enclosing->enclosingClass();
}
- fullClassName.prepend(packageName() + u'.');
+ fullClassName.prepend(metaClass->typeEntry()->targetLangPackage() + u'.');
return fullClassName;
}
@@ -666,12 +666,6 @@ QString ShibokenGenerator::cpythonTypeName(const TypeEntryCPtr &type)
return cpythonBaseName(type) + u"_TypeF()"_s;
}
-QString ShibokenGenerator::cpythonTypeNameExt(const TypeEntryCPtr &type)
-{
- return cppApiVariableName(type->targetLangPackage()) + u'['
- + getTypeIndexVariableName(type) + u']';
-}
-
QString ShibokenGenerator::converterObject(const AbstractMetaType &type)
{
if (type.isCString())
@@ -727,12 +721,32 @@ QString ShibokenGenerator::converterObject(const TypeEntryCPtr &type)
+ u'[' + getTypeIndexVariableName(type) + u']';
}
-QString ShibokenGenerator::cpythonTypeNameExt(const AbstractMetaType &type)
+QString ShibokenGenerator::cpythonTypeNameExtSet(const TypeEntryCPtr &type)
+{
+ return cppApiVariableName(type->targetLangPackage()) + u'['
+ + getTypeIndexVariableName(type) + u']';
+}
+
+QString ShibokenGenerator::cpythonTypeNameExtSet(const AbstractMetaType &type)
{
return cppApiVariableName(type.typeEntry()->targetLangPackage()) + u'['
+ getTypeIndexVariableName(type) + u']';
}
+QString ShibokenGenerator::cpythonTypeNameExt(const TypeEntryCPtr &type)
+{
+ return "Shiboken::Module::get("_L1 + cppApiVariableName(type->targetLangPackage())
+ + ", "_L1 + getTypeIndexVariableName(type) + ", \""_L1
+ + type->qualifiedTargetLangName() + "\")"_L1;
+}
+
+QString ShibokenGenerator::cpythonTypeNameExt(const AbstractMetaType &type)
+{
+ return "Shiboken::Module::get("_L1 + cppApiVariableName(type.typeEntry()->targetLangPackage())
+ + ", "_L1 + getTypeIndexVariableName(type) + ", \""_L1
+ + type.typeEntry()->qualifiedTargetLangName() + "\")"_L1;
+}
+
QString ShibokenGenerator::fixedCppTypeName(const TargetToNativeConversion &toNative)
{
if (toNative.sourceType())
diff --git a/sources/shiboken6/generator/shiboken/shibokengenerator.h b/sources/shiboken6/generator/shiboken/shibokengenerator.h
index 6c5355edb..eeff95a6c 100644
--- a/sources/shiboken6/generator/shiboken/shibokengenerator.h
+++ b/sources/shiboken6/generator/shiboken/shibokengenerator.h
@@ -252,8 +252,10 @@ protected:
static QString cpythonBaseName(const AbstractMetaType &type);
static QString cpythonTypeName(const AbstractMetaClassCPtr &metaClass);
static QString cpythonTypeName(const TypeEntryCPtr &type);
+ static QString cpythonTypeNameExtSet(const TypeEntryCPtr &type);
+ static QString cpythonTypeNameExtSet(const AbstractMetaType &type);
static QString cpythonTypeNameExt(const TypeEntryCPtr &type);
- static QString cpythonTypeNameExt(const AbstractMetaType &type) ;
+ static QString cpythonTypeNameExt(const AbstractMetaType &type);
static QString cpythonCheckFunction(TypeEntryCPtr type);
static QString cpythonCheckFunction(AbstractMetaType metaType);
static QString cpythonIsConvertibleFunction(const TypeEntryCPtr &type);
diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp
index 779cea5ed..35d46b1aa 100644
--- a/sources/shiboken6/libshiboken/basewrapper.cpp
+++ b/sources/shiboken6/libshiboken/basewrapper.cpp
@@ -10,6 +10,7 @@
#include "sbkenum.h"
#include "sbkerrors.h"
#include "sbkfeature_base.h"
+#include "sbkmodule.h"
#include "sbkstring.h"
#include "sbkstaticstrings.h"
#include "sbkstaticstrings_p.h"
diff --git a/sources/shiboken6/libshiboken/sbkconverter.cpp b/sources/shiboken6/libshiboken/sbkconverter.cpp
index 215190515..7df42a08e 100644
--- a/sources/shiboken6/libshiboken/sbkconverter.cpp
+++ b/sources/shiboken6/libshiboken/sbkconverter.cpp
@@ -4,6 +4,7 @@
#include "sbkconverter.h"
#include "sbkconverter_p.h"
#include "sbkarrayconverter_p.h"
+#include "sbkmodule.h"
#include "basewrapper_p.h"
#include "bindingmanager.h"
#include "autodecref.h"
@@ -409,9 +410,24 @@ void registerConverterName(SbkConverter *converter, const char *typeName)
converters.insert(std::make_pair(typeName, converter));
}
+static std::string getRealTypeName(const char *name)
+{
+ std::string typeName(name);
+ auto size = typeName.size();
+ if (std::isalnum(typeName[size - 1]) == 0)
+ return typeName.substr(0, size - 1);
+ return typeName;
+}
+
SbkConverter *getConverter(const char *typeName)
{
- ConvertersMap::const_iterator it = converters.find(typeName);
+ auto it = converters.find(typeName);
+ if (it != converters.end())
+ return it->second;
+ // PYSIDE-2404: Did not find the name. Load the lazy classes
+ // which have this name and try again.
+ Shiboken::Module::loadLazyClassesWithName(getRealTypeName(typeName).c_str());
+ it = converters.find(typeName);
if (it != converters.end())
return it->second;
if (Shiboken::pyVerbose() > 0) {
@@ -676,7 +692,14 @@ PyTypeObject *getPythonTypeObject(const SbkConverter *converter)
PyTypeObject *getPythonTypeObject(const char *typeName)
{
- return getPythonTypeObject(getConverter(typeName));
+ auto *type = getPythonTypeObject(getConverter(typeName));
+ if (type == nullptr) {
+ // PYSIDE-2404: Did not find the name. Load the lazy classes
+ // which have this name and try again.
+ Shiboken::Module::loadLazyClassesWithName(getRealTypeName(typeName).c_str());
+ type = getPythonTypeObject(getConverter(typeName));
+ }
+ return type;
}
bool pythonTypeIsValueType(const SbkConverter *converter)
diff --git a/sources/shiboken6/libshiboken/sbkcppstring.cpp b/sources/shiboken6/libshiboken/sbkcppstring.cpp
index 42b09111c..8e8324f5e 100644
--- a/sources/shiboken6/libshiboken/sbkcppstring.cpp
+++ b/sources/shiboken6/libshiboken/sbkcppstring.cpp
@@ -12,6 +12,11 @@ PyObject *fromCppString(const std::string &value)
return PyUnicode_FromStringAndSize(value.data(), value.size());
}
+PyObject *fromCppStringView(std::string_view value)
+{
+ return PyUnicode_FromStringAndSize(value.data(), value.size());
+}
+
PyObject *fromCppWString(const std::wstring &value)
{
return PyUnicode_FromWideChar(value.data(), value.size());
diff --git a/sources/shiboken6/libshiboken/sbkcppstring.h b/sources/shiboken6/libshiboken/sbkcppstring.h
index f418ea8dd..7ffe11c75 100644
--- a/sources/shiboken6/libshiboken/sbkcppstring.h
+++ b/sources/shiboken6/libshiboken/sbkcppstring.h
@@ -8,10 +8,12 @@
#include "shibokenmacros.h"
#include <string>
+#include <string_view>
namespace Shiboken::String
{
LIBSHIBOKEN_API PyObject *fromCppString(const std::string &value);
+ LIBSHIBOKEN_API PyObject *fromCppStringView(std::string_view value);
LIBSHIBOKEN_API PyObject *fromCppWString(const std::wstring &value);
LIBSHIBOKEN_API void toCppString(PyObject *str, std::string *value);
LIBSHIBOKEN_API void toCppWString(PyObject *str, std::wstring *value);
diff --git a/sources/shiboken6/libshiboken/sbkmodule.cpp b/sources/shiboken6/libshiboken/sbkmodule.cpp
index aeae34a36..6b06764b5 100644
--- a/sources/shiboken6/libshiboken/sbkmodule.cpp
+++ b/sources/shiboken6/libshiboken/sbkmodule.cpp
@@ -2,9 +2,15 @@
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "sbkmodule.h"
+#include "autodecref.h"
#include "basewrapper.h"
#include "bindingmanager.h"
+#include "sbkstring.h"
+#include "sbkcppstring.h"
+
#include <unordered_map>
+#include <unordered_set>
+#include <cstring>
/// This hash maps module objects to arrays of Python types.
using ModuleTypesMap = std::unordered_map<PyObject *, PyTypeObject **> ;
@@ -12,34 +18,441 @@ using ModuleTypesMap = std::unordered_map<PyObject *, PyTypeObject **> ;
/// This hash maps module objects to arrays of converters.
using ModuleConvertersMap = std::unordered_map<PyObject *, SbkConverter **>;
+/// This hash maps type names to type creation functions.
+using TypeCreationFunctionModulePair =
+ std::pair<Shiboken::Module::TypeCreationFunction, PyObject *>;
+using NameToTypeFunctionMap = std::unordered_map<std::string, TypeCreationFunctionModulePair>;
+
+/// This hash maps module objects to maps of names to functions.
+using ModuleToFuncsMap = std::unordered_map<PyObject *, NameToTypeFunctionMap> ;
+
/// All types produced in imported modules are mapped here.
static ModuleTypesMap moduleTypes;
static ModuleConvertersMap moduleConverters;
+static ModuleToFuncsMap moduleToFuncs;
namespace Shiboken
{
namespace Module
{
+// PYSIDE-2404: Replacing the arguments generated by cpythonTypeNameExt
+// by a function call.
+LIBSHIBOKEN_API PyTypeObject *get(PyTypeObject **types, int index, const char *typeName)
+{
+ if (types[index] != nullptr)
+ return types[index];
+
+ static PyObject *sysModules = PyImport_GetModuleDict();
+
+ // The slow path for initialization.
+ // We get the type by following the chain from the module.
+ // As soon as types[index] gets filled, we can stop.
+
+ std::string_view names(typeName);
+ bool usePySide = names.substr(0, 8) == std::string("PySide6.");
+ auto dotPos = usePySide ? names.find('.', 8) : names.find('.');
+ auto startPos = dotPos + 1;
+ AutoDecRef modName(String::fromCppStringView(names.substr(0, dotPos)));
+ auto *modOrType = PyDict_GetItem(sysModules, modName);
+ if (!modOrType)
+ modOrType = PyImport_Import(modName);
+
+ do {
+ dotPos = names.find('.', startPos);
+ auto typeName = dotPos != std::string::npos
+ ? names.substr(startPos, dotPos - startPos)
+ : names.substr(startPos);
+ startPos = dotPos + 1;
+ AutoDecRef obTypeName(String::fromCppStringView(typeName));
+ modOrType = PyObject_GetAttr(modOrType, obTypeName);
+ } while (types[index] == nullptr && dotPos != std::string::npos);
+
+ return types[index];
+}
+
+static PyTypeObject *incarnateType(PyObject *module, const char *name,
+ NameToTypeFunctionMap &nameToFunc)
+{
+ // - locate the name and retrieve the generating function
+ auto funcIter = nameToFunc.find(name);
+ if (funcIter == nameToFunc.end()) {
+ // attribute does really not exist.
+ return nullptr;
+ }
+ // - call this function that returns a PyTypeObject
+ auto pair = funcIter->second;
+ auto initFunc = pair.first;
+ auto *modOrType = pair.second;
+
+ // PYSIDE-2404: Make sure that no switching happens during type creation.
+ auto saveFeature = initSelectableFeature(nullptr);
+ PyTypeObject *type = initFunc(modOrType);
+ initSelectableFeature(saveFeature);
+
+ // - assign this object to the name in the module
+ auto *res = reinterpret_cast<PyObject *>(type);
+ Py_INCREF(res);
+ PyModule_AddObject(module, name, res); // steals reference
+ // - remove the entry, if not by something cleared.
+ if (!nameToFunc.empty())
+ nameToFunc.erase(funcIter);
+ // - return the PyTypeObject.
+ return type;
+}
+
+// PYSIDE-2404: Make sure that the mentioned classes really exist.
+// Used in `Pyside::typeName`. Because the result will be cached by
+// the creation of the type(s), this is efficient.
+void loadLazyClassesWithName(const char *name)
+{
+ for (auto const & tableIter : moduleToFuncs) {
+ auto nameToFunc = tableIter.second;
+ auto funcIter = nameToFunc.find(name);
+ if (funcIter != nameToFunc.end()) {
+ // attribute exists in the lazy types.
+ auto *module = tableIter.first;
+ incarnateType(module, name, nameToFunc);
+ }
+ }
+}
+
+// PYSIDE-2404: Completely load all not yet loaded classes.
+// This is needed to resolve a star import.
+void resolveLazyClasses(PyObject *module)
+{
+ // - locate the module in the moduleTofuncs mapping
+ auto tableIter = moduleToFuncs.find(module);
+ if (tableIter == moduleToFuncs.end())
+ return;
+
+ // - see if there are still unloaded elements
+ auto &nameToFunc = tableIter->second;
+
+ // - incarnate all types.
+ while (!nameToFunc.empty()) {
+ auto it = nameToFunc.begin();
+ auto attrNameStr = it->first;
+ incarnateType(module, attrNameStr.c_str(), nameToFunc);
+ }
+}
+
+// PYSIDE-2404: Use module getattr to do on-demand initialization.
+static PyObject *_module_getattr_template(PyObject * /* self */, PyObject *args)
+{
+ // An attribute was not found. Look it up in the shadow dict, resolve it
+ // and put it into the module dict afterwards.
+ PyObject *module{};
+ PyObject *attrName{};
+ if (!PyArg_ParseTuple(args, "OO", &module, &attrName))
+ return nullptr;
+
+ // - locate the module in the moduleTofuncs mapping
+ auto tableIter = moduleToFuncs.find(module);
+ assert(tableIter != moduleToFuncs.end());
+ // - locate the name and retrieve the generating function
+ const char *attrNameStr = Shiboken::String::toCString(attrName);
+ auto &nameToFunc = tableIter->second;
+
+ auto *type = incarnateType(module, attrNameStr, nameToFunc);
+ auto *ret = reinterpret_cast<PyObject *>(type);
+ if (ret == nullptr) // attribute does really not exist. Should not happen.
+ PyErr_SetNone(PyExc_AttributeError);
+ return ret;
+}
+
+// PYSIDE-2404: Supply a new module dir for not yet visible entries.
+static PyObject *_module_dir_template(PyObject * /* self */, PyObject *args)
+{
+ static PyObject *const _dict = Shiboken::String::createStaticString("__dict__");
+ // The dir function must replace all of the builtin function.
+ PyObject *module{};
+ if (!PyArg_ParseTuple(args, "O", &module))
+ return nullptr;
+
+ auto tableIter = moduleToFuncs.find(module);
+ assert(tableIter != moduleToFuncs.end());
+ Shiboken::AutoDecRef dict(PyObject_GetAttr(module, _dict));
+ auto *ret = PyDict_Keys(dict);
+ // Now add all elements that were not yet in the dict.
+ auto &nameToFunc = tableIter->second;
+ for (const auto &funcIter : nameToFunc) {
+ const char *name = funcIter.first.c_str();
+ Shiboken::AutoDecRef pyName(PyUnicode_FromString(name));
+ PyList_Append(ret, pyName);
+ }
+ return ret;
+}
+
+PyMethodDef module_methods[] = {
+ {"__getattr__", (PyCFunction)_module_getattr_template, METH_VARARGS, nullptr},
+ {"__dir__", (PyCFunction)_module_dir_template, METH_VARARGS, nullptr},
+ {nullptr, nullptr, 0, nullptr}
+};
+
+// Python 3.8 - 3.12
+static int const LOAD_CONST_312 = 100;
+static int const IMPORT_NAME_312 = 108;
+
+static bool isImportStar(PyObject *module)
+{
+ // Find out whether we have a star import. This must work even
+ // when we have no import support from feature.
+ static PyObject *const _f_code = Shiboken::String::createStaticString("f_code");
+ static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti");
+ static PyObject *const _f_back = Shiboken::String::createStaticString("f_back");
+ static PyObject *const _co_code = Shiboken::String::createStaticString("co_code");
+ static PyObject *const _co_consts = Shiboken::String::createStaticString("co_consts");
+ static PyObject *const _co_names = Shiboken::String::createStaticString("co_names");
+
+ auto *obFrame = reinterpret_cast<PyObject *>(PyEval_GetFrame());
+ if (obFrame == nullptr)
+ return true; // better assume worst-case.
+
+ Py_INCREF(obFrame);
+ AutoDecRef dec_frame(obFrame);
+
+ // Calculate the offset of the running import_name opcode on the stack.
+ // Right before that there must be a load_const with the tuple `("*",)`.
+ while (dec_frame.object() != Py_None) {
+ AutoDecRef dec_f_code(PyObject_GetAttr(dec_frame, _f_code));
+ AutoDecRef dec_co_code(PyObject_GetAttr(dec_f_code, _co_code));
+ AutoDecRef dec_f_lasti(PyObject_GetAttr(dec_frame, _f_lasti));
+ Py_ssize_t f_lasti = PyLong_AsSsize_t(dec_f_lasti);
+ Py_ssize_t code_len;
+ char *co_code{};
+ PyBytes_AsStringAndSize(dec_co_code, &co_code, &code_len);
+ uint8_t opcode2 = co_code[f_lasti];
+ uint8_t opcode1 = co_code[f_lasti - 2];
+ if (opcode1 == LOAD_CONST_312 && opcode2 == IMPORT_NAME_312) {
+ uint8_t oparg1 = co_code[f_lasti - 1];
+ uint8_t oparg2 = co_code[f_lasti + 1];
+ AutoDecRef dec_co_consts(PyObject_GetAttr(dec_f_code, _co_consts));
+ auto *fromlist = PyTuple_GetItem(dec_co_consts, oparg1);
+ if (PyTuple_Check(fromlist) && PyTuple_Size(fromlist) == 1
+ && Shiboken::String::toCString(PyTuple_GetItem(fromlist, 0))[0] == '*') {
+ AutoDecRef dec_co_names(PyObject_GetAttr(dec_f_code, _co_names));
+ const char *name = String::toCString(PyTuple_GetItem(dec_co_names, oparg2));
+ const char *modName = PyModule_GetName(module);
+ if (std::strcmp(name, modName) == 0)
+ return true;
+ }
+ }
+ dec_frame.reset(PyObject_GetAttr(dec_frame, _f_back));
+ }
+ return false;
+}
+
+// PYSIDE-2404: These modules produce ambiguous names which we cannot handle, yet.
+static std::unordered_set<std::string> dontLazyLoad{
+ "sample",
+ "smart",
+ "testbinding"
+};
+
+static const std::unordered_set<std::string> knownModules{
+ "shiboken6.Shiboken",
+ "minimal",
+ "other",
+ "sample",
+ "smart",
+ "scriptableapplication",
+ "testbinding"
+};
+
+static bool canNotLazyLoad(PyObject *module)
+{
+ const char *modName = PyModule_GetName(module);
+
+ // There are no more things that must be disabled :-D
+ return dontLazyLoad.find(modName) != dontLazyLoad.end();
+}
+
+static bool shouldLazyLoad(PyObject *module)
+{
+ const char *modName = PyModule_GetName(module);
+
+ if (knownModules.find(modName) != knownModules.end())
+ return true;
+ return std::strncmp(modName, "PySide6.", 8) == 0;
+}
+
+void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func)
+{
+ // - locate the module in the moduleTofuncs mapping
+ auto tableIter = moduleToFuncs.find(module);
+ assert(tableIter != moduleToFuncs.end());
+ // - Assign the name/generating function pair.
+ auto &nameToFunc = tableIter->second;
+ TypeCreationFunctionModulePair pair{func, module};
+ auto nit = nameToFunc.find(name);
+ if (nit == nameToFunc.end())
+ nameToFunc.insert(std::make_pair(name, pair));
+ else
+ nit->second = pair;
+
+ // PYSIDE-2404: Lazy Loading
+ //
+ // Options:
+ // 0 - switch lazy loading off.
+ // 1 - lazy loading for all known modules.
+ // 3 - lazy loading for any module.
+ //
+ // By default we lazy load all known modules (option = 1).
+ const auto *flag = getenv("PYSIDE6_OPTION_LAZY");
+ const int value = flag != nullptr ? std::atoi(flag) : 1;
+
+ if (value == 0 // completely disabled
+ || canNotLazyLoad(module) // for some reason we cannot lazy load
+ || (value == 1 && !shouldLazyLoad(module)) // not a known module
+ ) {
+ PyTypeObject *type = func(module);
+ PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(type)); // steals reference
+ }
+}
+
+void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func,
+ const char *containerName)
+{
+ // This version could be delayed as well, but for the few cases
+ // we simply fetch the container type and insert directly.
+ AutoDecRef obContainerType(PyObject_GetAttrString(module, containerName));
+ PyTypeObject *type = func(obContainerType);
+ PyObject_SetAttrString(obContainerType, name, reinterpret_cast<PyObject *>(type)); // steals reference
+}
+
+void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func,
+ const char *outerContainerName,
+ const char *innerContainerName)
+{
+ // This version has even more indirection. It is very rare, and
+ // we handle it directly.
+ AutoDecRef obOuterType(PyObject_GetAttrString(module, outerContainerName));
+ AutoDecRef obInnerType(PyObject_GetAttrString(obOuterType, innerContainerName));
+ PyTypeObject *type = func(obInnerType);
+ PyObject_SetAttrString(obInnerType, name, reinterpret_cast<PyObject *>(type)); // steals reference
+}
+
+void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func,
+ const char *containerName3,
+ const char *containerName2,
+ const char *containerName)
+{
+ // This version has even mode indirection. It is very rare, and
+ // we handle it directly.
+ AutoDecRef obContainerType3(PyObject_GetAttrString(module, containerName3));
+ AutoDecRef obContainerType2(PyObject_GetAttrString(obContainerType3, containerName2));
+ AutoDecRef obContainerType(PyObject_GetAttrString(obContainerType2, containerName));
+ PyTypeObject *type = func(obContainerType);
+ PyObject_SetAttrString(obContainerType, name, reinterpret_cast<PyObject *>(type)); // steals reference
+}
+
PyObject *import(const char *moduleName)
{
PyObject *sysModules = PyImport_GetModuleDict();
PyObject *module = PyDict_GetItemString(sysModules, moduleName);
- if (module)
+ if (module != nullptr)
Py_INCREF(module);
else
module = PyImport_ImportModule(moduleName);
- if (!module)
- PyErr_Format(PyExc_ImportError,"could not import module '%s'", moduleName);
+ if (module == nullptr)
+ PyErr_Format(PyExc_ImportError, "could not import module '%s'", moduleName);
return module;
}
-PyObject *create(const char * /* moduleName */, void *moduleData)
+// PYSIDE-2404: Redirecting import for "import *" support.
+//
+// The first import will be handled by the isImportStar function.
+// But the same module might be imported twice, which would give no
+// introspection due to module caching.
+
+static PyObject *origImportFunc{};
+
+static PyObject *lazy_import(PyObject * /* self */, PyObject *args, PyObject *kwds)
+{
+ auto *ret = PyObject_Call(origImportFunc, args, kwds);
+ if (ret != nullptr) {
+ // PYSIDE-2404: Support star import when lazy loading.
+ if (PyTuple_Size(args) >= 4) {
+ auto *fromlist = PyTuple_GetItem(args, 3);
+ if (PyTuple_Check(fromlist) && PyTuple_Size(fromlist) == 1
+ && Shiboken::String::toCString(PyTuple_GetItem(fromlist, 0))[0] == '*')
+ Shiboken::Module::resolveLazyClasses(ret);
+ }
+ }
+ return ret;
+}
+
+static PyMethodDef lazy_methods[] = {
+ {"__lazy_import__", (PyCFunction)lazy_import, METH_VARARGS | METH_KEYWORDS, nullptr},
+ {nullptr, nullptr, 0, nullptr}
+};
+
+// PYSIDE-2404: Nuitka is stealing our `__getattr__` entry from the
+// module dicts. Until we remove this vulnerability from
+// our modules, we disable Lazy Init when Nuitka is present.
+static bool isNuitkaPresent()
+{
+ static PyObject *const sysModules = PyImport_GetModuleDict();
+ static PyObject *const compiled = Shiboken::String::createStaticString("__compiled__");
+
+ PyObject *key{}, *value{};
+ Py_ssize_t pos = 0;
+ while (PyDict_Next(sysModules, &pos, &key, &value)) {
+ if (PyObject_HasAttr(value, compiled))
+ return true;
+ }
+ return false;
+}
+
+PyObject *create(const char * /* modName */, void *moduleData)
{
+ static auto *builtins = PyEval_GetBuiltins();
+ static auto *partial = Pep_GetPartialFunction();
+ static bool lazy_init{};
+ static bool nuitkaPresent = isNuitkaPresent();
+
Shiboken::init();
- return PyModule_Create(reinterpret_cast<PyModuleDef *>(moduleData));
+ auto *module = PyModule_Create(reinterpret_cast<PyModuleDef *>(moduleData));
+
+ // Setup of a getattr function for "missing" classes and a dir replacement.
+ for (int idx = 0; module_methods[idx].ml_name != nullptr; ++idx) {
+ auto *pyFuncPlus = PyCFunction_NewEx(&module_methods[idx], nullptr, nullptr);
+ // Turn this function into a bound object, so we have access to the module.
+ auto *pyFunc = PyObject_CallFunctionObjArgs(partial, pyFuncPlus, module, nullptr);
+ PyModule_AddObject(module, module_methods[idx].ml_name, pyFunc); // steals reference
+ }
+ // Insert an initial empty table for the module.
+ NameToTypeFunctionMap empty;
+ moduleToFuncs.insert(std::make_pair(module, empty));
+
+ // A star import must be done unconditionally. Use the complete name.
+ if (isImportStar(module))
+ dontLazyLoad.insert(PyModule_GetName(module));
+
+ // For now, we also need to disable Lazy Init when Nuitka is there.
+ if (nuitkaPresent)
+ dontLazyLoad.insert(PyModule_GetName(module));
+
+ // Also add the lazy import redirection.
+ if (!lazy_init) {
+ origImportFunc = PyDict_GetItemString(builtins, "__import__");
+ // The single function to be called.
+ auto *func = PyCFunction_NewEx(lazy_methods, nullptr, nullptr);
+ PyDict_SetItemString(builtins, "__import__", func);
+ lazy_init = true;
+ }
+ return module;
}
void registerTypes(PyObject *module, PyTypeObject **types)
diff --git a/sources/shiboken6/libshiboken/sbkmodule.h b/sources/shiboken6/libshiboken/sbkmodule.h
index c39692f17..5cc5dc47b 100644
--- a/sources/shiboken6/libshiboken/sbkmodule.h
+++ b/sources/shiboken6/libshiboken/sbkmodule.h
@@ -14,6 +14,15 @@ struct SbkConverter;
namespace Shiboken::Module {
+/// PYSIDE-2404: Replacing the arguments in cpythonTypeNameExt by a function.
+LIBSHIBOKEN_API PyTypeObject *get(PyTypeObject **types, int index, const char *typeName);
+
+/// PYSIDE-2404: Make sure that mentioned classes really exist.
+LIBSHIBOKEN_API void loadLazyClassesWithName(const char *name);
+
+/// PYSIDE-2404: incarnate all classes for star imports.
+LIBSHIBOKEN_API void resolveLazyClasses(PyObject *module);
+
/**
* Imports and returns the module named \p moduleName, or a NULL pointer in case of failure.
* If the module is already imported, it increments its reference count before returning it.
@@ -29,6 +38,30 @@ LIBSHIBOKEN_API PyObject *import(const char *moduleName);
*/
LIBSHIBOKEN_API PyObject *create(const char *moduleName, void *moduleData);
+using TypeCreationFunction = PyTypeObject *(*)(PyObject *module);
+
+/// Adds a type creation function to the module.
+LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func);
+
+LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func,
+ const char *containerName);
+
+LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func,
+ const char *outerContainerName,
+ const char *innerContainerName);
+
+LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module,
+ const char *name,
+ TypeCreationFunction func,
+ const char *containerName3,
+ const char *containerName2,
+ const char *containerName);
/**
* Registers the list of types created by \p module.
* \param module Module where the types were created.
diff --git a/sources/shiboken6/libshiboken/signature/signature.cpp b/sources/shiboken6/libshiboken/signature/signature.cpp
index 04eb974f0..3255cb56d 100644
--- a/sources/shiboken6/libshiboken/signature/signature.cpp
+++ b/sources/shiboken6/libshiboken/signature/signature.cpp
@@ -474,6 +474,8 @@ void FinishSignatureInitialization(PyObject *module, const char *signatures[])
* Still, it is not possible to call init phase 2 from here,
* because the import is still running. Do it from Python!
*/
+ init_shibokensupport_module();
+
#ifndef PYPY_VERSION
static const bool patch_types = true;
#else