diff options
28 files changed, 494 insertions, 236 deletions
diff --git a/build_history/blacklist.txt b/build_history/blacklist.txt index 9b63f9784..2a2a5d4c4 100644 --- a/build_history/blacklist.txt +++ b/build_history/blacklist.txt @@ -18,9 +18,6 @@ darwin py3 [QtCore::qfileread_test] darwin -# Nested exception in Python 3 -[QtCore::qflags_test] - py3 [QtCore::qobject_connect_notify_test] linux darwin diff --git a/build_scripts/platforms/windows_desktop.py b/build_scripts/platforms/windows_desktop.py index 345847e20..6b92c0823 100644 --- a/build_scripts/platforms/windows_desktop.py +++ b/build_scripts/platforms/windows_desktop.py @@ -270,7 +270,12 @@ def copy_msvc_redist_files(vars, redist_target_path): zip_file = "pyside_qt_deps_64_2019.7z" if "{target_arch}".format(**vars) == "32": zip_file = "pyside_qt_deps_32_2019.7z" - download_and_extract_7z(redist_url + zip_file, redist_target_path) + try: + download_and_extract_7z(redist_url + zip_file, redist_target_path) + except: + print("download.qt.io is down, try with mirror") + redist_url = "https://www.funet.fi/pub/mirrors/download.qt-project.org/development_releases/prebuilt/vcredist/" + download_and_extract_7z(redist_url + zip_file, redist_target_path) else: print("Qt dependency DLLs (MSVC redist) will not be downloaded and extracted.") diff --git a/build_scripts/setup_runner.py b/build_scripts/setup_runner.py index 6eb0d8a26..3c7116b82 100644 --- a/build_scripts/setup_runner.py +++ b/build_scripts/setup_runner.py @@ -93,7 +93,7 @@ class SetupRunner(object): command = self.sub_argv[1] # Add --reuse-build option if requested and not already present. - if (reuse_build and command != 'clean' + if (reuse_build and command in ('bdist_wheel', 'build', 'build_rst_docs', 'install') and not self.cmd_line_argument_is_in_args("reuse-build", self.sub_argv)): setup_cmd.append(self.construct_cmd_line_argument("reuse-build")) self.invocations_list.append(setup_cmd) diff --git a/build_scripts/utils.py b/build_scripts/utils.py index 0782ae036..002d6ae5b 100644 --- a/build_scripts/utils.py +++ b/build_scripts/utils.py @@ -691,6 +691,16 @@ def find_llvm_config(): return result +# Expand the __ARCH_ place holder in the CLANG environment variables +def expand_clang_variables(target_arch): + for var in 'LLVM_INSTALL_DIR', 'CLANG_INSTALL_DIR': + value = os.environ.get(var) + if value and '_ARCH_' in value: + value = value.replace('_ARCH_', target_arch) + os.environ[var] = value + print("{} = {}".format(var, value)) + + # Add Clang to path for Windows for the shiboken ApiExtractor tests. # Revisit once Clang is bundled with Qt. def detect_clang(): diff --git a/coin/instructions/common_environment.yaml b/coin/instructions/common_environment.yaml index 2c6603334..41ab0059c 100644 --- a/coin/instructions/common_environment.yaml +++ b/coin/instructions/common_environment.yaml @@ -115,3 +115,10 @@ instructions: condition: property property: target.compiler equals_value: ICC_18 + - type: EnvironmentVariable + variableName: LLVM_INSTALL_DIR + variableValue: "{{.Env.LLVM_DYNAMIC_LIBS_100}}" + disable_if: + condition: property + property: host.osVersion + equals_value: openSUSE_15_1 diff --git a/coin_build_instructions.py b/coin_build_instructions.py index 95d800b56..d1e578fd7 100644 --- a/coin_build_instructions.py +++ b/coin_build_instructions.py @@ -38,7 +38,7 @@ ############################################################################# from build_scripts.options import has_option from build_scripts.options import option_value -from build_scripts.utils import install_pip_dependencies +from build_scripts.utils import install_pip_dependencies, expand_clang_variables from build_scripts.utils import get_qtci_virtualEnv from build_scripts.utils import run_instruction from build_scripts.utils import rmtree @@ -175,6 +175,8 @@ def run_build_instructions(phase): if __name__ == "__main__": # Remove some environment variables that impact cmake + arch = '32' if CI_TARGET_ARCH and CI_TARGET_ARCH == 'X86' else '64' + expand_clang_variables(arch) for env_var in ['CC', 'CXX']: if os.environ.get(env_var): del os.environ[env_var] diff --git a/coin_test_instructions.py b/coin_test_instructions.py index 467f58d19..3b8f7e39c 100644 --- a/coin_test_instructions.py +++ b/coin_test_instructions.py @@ -38,7 +38,7 @@ ############################################################################# from build_scripts.options import has_option from build_scripts.options import option_value -from build_scripts.utils import install_pip_dependencies +from build_scripts.utils import install_pip_dependencies, expand_clang_variables from build_scripts.utils import get_qtci_virtualEnv from build_scripts.utils import run_instruction from build_scripts.utils import rmtree @@ -103,6 +103,8 @@ def call_testrunner(python_ver, buildnro): def run_test_instructions(): # Remove some environment variables that impact cmake + arch = '32' if CI_TARGET_ARCH and CI_TARGET_ARCH == 'X86' else '64' + expand_clang_variables(arch) for env_var in ['CC', 'CXX']: if os.environ.get(env_var): del os.environ[env_var] diff --git a/product_dependencies.yaml b/product_dependencies.yaml deleted file mode 100644 index 29c4fb5d6..000000000 --- a/product_dependencies.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: - ../../qt/qt5.git: - ref: "5.15.2" diff --git a/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml b/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml index 205e4ef06..e205213e6 100644 --- a/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml +++ b/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml @@ -112,7 +112,6 @@ <rejection class="QtPrivate"/> <rejection class="QtSharedPointer"/> <rejection class="QtStringBuilder"/> - <rejection class="std"/> <rejection class="QByteArray" field-name="MaxSize"/> <rejection class="QChildEvent" field-name="c"/> diff --git a/sources/pyside2/PySide2/QtTest/CMakeLists.txt b/sources/pyside2/PySide2/QtTest/CMakeLists.txt index 04bb28cc9..50b9e3e30 100644 --- a/sources/pyside2/PySide2/QtTest/CMakeLists.txt +++ b/sources/pyside2/PySide2/QtTest/CMakeLists.txt @@ -2,6 +2,7 @@ project(QtTest) set(QtTest_SRC ${QtTest_GEN_DIR}/qtest_pysideqtoucheventsequence_wrapper.cpp +${QtTest_GEN_DIR}/qabstractitemmodeltester_wrapper.cpp ${QtTest_GEN_DIR}/qtest_wrapper.cpp # module is always needed ${QtTest_GEN_DIR}/qttest_module_wrapper.cpp diff --git a/sources/pyside2/PySide2/QtTest/typesystem_test.xml b/sources/pyside2/PySide2/QtTest/typesystem_test.xml index f7facaf7d..5aa45febb 100644 --- a/sources/pyside2/PySide2/QtTest/typesystem_test.xml +++ b/sources/pyside2/PySide2/QtTest/typesystem_test.xml @@ -102,6 +102,10 @@ </object-type> ^^^ this is now moved into QtGui --> + <object-type name="QAbstractItemModelTester"> + <enum-type name="FailureReportingMode"/> + </object-type> + <namespace-type name="QTest"> <!-- Qt5: private <enum-type name="AttributeIndex" since="4.6"/> --> <enum-type name="KeyAction"/> diff --git a/sources/pyside2/libpyside/pyside.cpp b/sources/pyside2/libpyside/pyside.cpp index 297e5a3b2..5d0859adc 100644 --- a/sources/pyside2/libpyside/pyside.cpp +++ b/sources/pyside2/libpyside/pyside.cpp @@ -58,7 +58,6 @@ #include <sbkconverter.h> #include <sbkstring.h> #include <sbkstaticstrings.h> -#include <qapp_macro.h> #include <QtCore/QByteArray> #include <QtCore/QCoreApplication> @@ -304,6 +303,9 @@ void initQApp() */ if (!qApp) Py_DECREF(MakeQAppWrapper(nullptr)); + + // PYSIDE-1470: Register a function to destroy an application from shiboken. + setDestroyQApplication(destroyQCoreApplication); } PyObject *getMetaDataFromQObject(QObject *cppSelf, PyObject *self, PyObject *name) diff --git a/sources/pyside2/tests/QtCore/qflags_test.py b/sources/pyside2/tests/QtCore/qflags_test.py index 08a7c55b1..e1e989c1e 100644 --- a/sources/pyside2/tests/QtCore/qflags_test.py +++ b/sources/pyside2/tests/QtCore/qflags_test.py @@ -30,6 +30,7 @@ '''Test cases for QFlags''' +import operator import os import sys import unittest @@ -117,12 +118,13 @@ class QFlagsOnQVariant(unittest.TestCase): class QFlagsWrongType(unittest.TestCase): def testWrongType(self): '''Wrong type passed to QFlags binary operators''' + for op in operator.or_, operator.and_, operator.xor: + for x in '43', 'jabba', QObject, object: + self.assertRaises(TypeError, op, Qt.NoItemFlags, x) + self.assertRaises(TypeError, op, x, Qt.NoItemFlags) + # making sure this actually does not fail all the time + self.assertEqual(operator.or_(Qt.NoItemFlags, 43), 43) - self.assertRaises(TypeError, Qt.NoItemFlags | '43') - self.assertRaises(TypeError, Qt.NoItemFlags & '43') - self.assertRaises(TypeError, 'jabba' & Qt.NoItemFlags) - self.assertRaises(TypeError, 'hut' & Qt.NoItemFlags) - self.assertRaises(TypeError, Qt.NoItemFlags & QObject()) if __name__ == '__main__': unittest.main() diff --git a/sources/shiboken2/generator/generator.cpp b/sources/shiboken2/generator/generator.cpp index dd56ab7cd..f90cd312f 100644 --- a/sources/shiboken2/generator/generator.cpp +++ b/sources/shiboken2/generator/generator.cpp @@ -276,24 +276,24 @@ void Generator::addInstantiatedContainersAndSmartPointers(const AbstractMetaType return; } - QString typeName = getSimplifiedContainerTypeName(type); if (isContainer) { + QString typeName = getSimplifiedContainerTypeName(type); if (!m_d->instantiatedContainersNames.contains(typeName)) { m_d->instantiatedContainersNames.append(typeName); m_d->instantiatedContainers.append(type); } - } else { - // Is smart pointer. Check if the (const?) pointee is already known - auto pt = pointeeTypeEntry(type); - const bool present = - std::any_of(m_d->instantiatedSmartPointers.cbegin(), m_d->instantiatedSmartPointers.cend(), - [pt] (const AbstractMetaType *t) { - return pointeeTypeEntry(t) == pt; - }); - if (!present) - m_d->instantiatedSmartPointers.append(canonicalSmartPtrInstantiation(type)); + return; } - + // Is smart pointer. Check if the (const?) pointee is already known for the given + // smart pointer type entry. + auto pt = pointeeTypeEntry(type); + const bool present = + std::any_of(m_d->instantiatedSmartPointers.cbegin(), m_d->instantiatedSmartPointers.cend(), + [typeEntry, pt] (const AbstractMetaType *t) { + return t->typeEntry() == typeEntry && pointeeTypeEntry(t) == pt; + }); + if (!present) + m_d->instantiatedSmartPointers.append(canonicalSmartPtrInstantiation(type)); } void Generator::collectInstantiatedContainersAndSmartPointers(const AbstractMetaFunction *func) diff --git a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp index ff44db955..786308023 100644 --- a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp +++ b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp @@ -359,7 +359,6 @@ void CppGenerator::generateClass(QTextStream &s, const GeneratorContext &classCo << "#include <pyside.h>\n" << "#include <pysideqenum.h>\n" << "#include <feature_select.h>\n" - << "#include <qapp_macro.h>\n\n" << "QT_WARNING_DISABLE_DEPRECATED\n\n"; } @@ -1979,7 +1978,8 @@ void CppGenerator::writeConstructorWrapper(QTextStream &s, const AbstractMetaFun Indentation indent(INDENT); writeCodeSnips(s, func->injectedCodeSnips(), TypeSystem::CodeSnipPositionEnd, TypeSystem::TargetLangCode, func); } - s << INDENT << "}\n"; + s << INDENT << "}\n" + << INDENT << "break;\n"; break; } } @@ -4193,15 +4193,14 @@ void CppGenerator::writeClassDefinition(QTextStream &s, tp_new = QLatin1String("SbkDummyNew /* PYSIDE-595: Prevent replacement " "of \"0\" with base->tp_new. */"); } - tp_flags.append(QLatin1String("|Py_TPFLAGS_HAVE_GC")); } else if (isQApp) { tp_new = QLatin1String("SbkQAppTpNew"); // PYSIDE-571: need singleton app } else { tp_new = QLatin1String("SbkObjectTpNew"); - tp_flags.append(QLatin1String("|Py_TPFLAGS_HAVE_GC")); } + tp_flags.append(QLatin1String("|Py_TPFLAGS_HAVE_GC")); QString tp_richcompare; if (!metaClass->isNamespace() && metaClass->hasComparisonOperatorOverload()) @@ -5230,6 +5229,8 @@ void CppGenerator::writeFlagsBinaryOperator(QTextStream &s, const AbstractMetaEn s << INDENT << "cppArg = static_cast<" << flagsEntry->originalName() << ">(int(PyInt_AsLong(" << PYTHON_ARG << ")));\n"; s << "#endif\n\n"; + s << INDENT << "if (PyErr_Occurred())\n" << indent(INDENT) + << INDENT << "return nullptr;\n" << outdent(INDENT); s << INDENT << "cppResult = " << CPP_SELF_VAR << " " << cppOpName << " cppArg;\n"; s << INDENT << "return "; writeToPythonConversion(s, flagsType, nullptr, QLatin1String("cppResult")); @@ -5967,7 +5968,6 @@ bool CppGenerator::finishGeneration() s << "#include <pyside.h>\n"; s << "#include <pysideqenum.h>\n"; s << "#include <feature_select.h>\n"; - s << "#include <qapp_macro.h>\n"; } s << "#include \"" << getModuleHeaderFileName() << '"' << Qt::endl << Qt::endl; diff --git a/sources/shiboken2/libshiboken/CMakeLists.txt b/sources/shiboken2/libshiboken/CMakeLists.txt index a209dc711..45b41fd13 100644 --- a/sources/shiboken2/libshiboken/CMakeLists.txt +++ b/sources/shiboken2/libshiboken/CMakeLists.txt @@ -57,7 +57,6 @@ sbkstaticstrings.cpp bindingmanager.cpp threadstatesaver.cpp shibokenbuffer.cpp -qapp_macro.cpp pep384impl.cpp voidptr.cpp typespec.cpp @@ -142,7 +141,6 @@ install(FILES shibokenbuffer.h sbkpython.h pep384impl.h - qapp_macro.h voidptr.h typespec.h bufferprocs_py37.h diff --git a/sources/shiboken2/libshiboken/basewrapper.cpp b/sources/shiboken2/libshiboken/basewrapper.cpp index 4b1e6e564..dca5631bc 100644 --- a/sources/shiboken2/libshiboken/basewrapper.cpp +++ b/sources/shiboken2/libshiboken/basewrapper.cpp @@ -56,7 +56,6 @@ #include <algorithm> #include "threadstatesaver.h" #include "signature.h" -#include "qapp_macro.h" #include "voidptr.h" #include <iostream> @@ -96,6 +95,13 @@ static void SbkObjectTypeDealloc(PyObject *pyObj); static PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds); static SelectableFeatureHook SelectFeatureSet = nullptr; +static DestroyQAppHook DestroyQApplication = nullptr; + +// PYSIDE-1470: Provide a hook to kill an Application from Shiboken. +void setDestroyQApplication(DestroyQAppHook func) +{ + DestroyQApplication = func; +} static PyObject *Sbk_TypeGet___dict__(PyTypeObject *type, void *context); // forward @@ -319,6 +325,11 @@ static int SbkObject_traverse(PyObject *self, visitproc visit, void *arg) if (sbkSelf->ob_dict) Py_VISIT(sbkSelf->ob_dict); + +#if PY_VERSION_HEX >= 0x03090000 + // This was not needed before Python 3.9 (Python issue 35810 and 40217) + Py_VISIT(Py_TYPE(self)); +#endif return 0; } @@ -422,9 +433,7 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) // be invoked and it trying to delete this object while it is still in // progress from the first time around, resulting in a double delete and a // crash. - // PYSIDE-571: Some objects do not use GC, so check this! - if (PyObject_IS_GC(pyObj)) - PyObject_GC_UnTrack(pyObj); + PyObject_GC_UnTrack(pyObj); // Check that Python is still initialized as sometimes this is called by a static destructor // after Python interpeter is shutdown. @@ -536,6 +545,47 @@ void SbkObjectTypeDealloc(PyObject *pyObj) } } +//////////////////////////////////////////////////////////////////////////// +// +// Support for the qApp macro. +// +// qApp is a macro in Qt5. In Python, we simulate that a little by a +// variable that monitors Q*Application.instance(). +// This variable is also able to destroy the app by qApp.shutdown(). +// + +PyObject *MakeQAppWrapper(PyTypeObject *type) +{ + static PyObject *qApp_last = nullptr; + + // protecting from multiple application instances + if (!(type == nullptr || qApp_last == Py_None)) { + const char *res_name = PepType_GetNameStr(Py_TYPE(qApp_last)); + const char *type_name = PepType_GetNameStr(type); + PyErr_Format(PyExc_RuntimeError, "Please destroy the %s singleton before" + " creating a new %s instance.", res_name, type_name); + return nullptr; + } + + // monitoring the last application state + PyObject *qApp_curr = type != nullptr ? PyObject_GC_New(PyObject, type) : Py_None; + static PyObject *builtins = PyEval_GetBuiltins(); + if (PyDict_SetItem(builtins, Shiboken::PyName::qApp(), qApp_curr) < 0) + return nullptr; + qApp_last = qApp_curr; + // Note: This Py_INCREF would normally be wrong because the qApp + // object already has a reference from PyObject_GC_New. But this is + // exactly the needed reference that keeps qApp alive from alone! + Py_INCREF(qApp_curr); + // PYSIDE-1470: As a side effect, the interactive "_" variable tends to + // create reference cycles. It was found when using gc.collect(). But using + // PyGC_collect() inside the C code had no effect in the interactive shell. + // The cycle exists only in the eval loop of the interpreter! + if (PyDict_GetItem(builtins, Shiboken::PyName::underscore())) + PyDict_SetItem(builtins, Shiboken::PyName::underscore(), Py_None); + return qApp_curr; +} + ////////////////////////////////////////////////////////////////////////////// // // PYSIDE-1019: Support switchable extensions @@ -550,9 +600,11 @@ void SbkObjectTypeDealloc(PyObject *pyObj) // SbkObject_GenericSetAttr PyObject_GenericSetAttr // -void initSelectableFeature(SelectableFeatureHook func) +SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func) { + auto ret = SelectFeatureSet; SelectFeatureSet = func; + return ret; } static PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) @@ -574,7 +626,7 @@ static PyObject *Sbk_TypeGet___dict__(PyTypeObject *type, void *context) * This is the override for getting a dict. */ auto dict = type->tp_dict; - if (dict == NULL) + if (dict == nullptr) Py_RETURN_NONE; if (SelectFeatureSet != nullptr) dict = SelectFeatureSet(type); @@ -718,10 +770,13 @@ static PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyOb sotp->d_func = nullptr; sotp->is_user_type = 1; + // PYSIDE-1463: Prevent feature switching while in the creation process + auto saveFeature = initSelectableFeature(nullptr); for (SbkObjectType *base : bases) { if (PepType_SOTP(base)->subtype_init) PepType_SOTP(base)->subtype_init(newType, args, kwds); } + initSelectableFeature(saveFeature); return reinterpret_cast<PyObject *>(newType); } @@ -744,39 +799,25 @@ static PyObject *_setupNew(SbkObject *self, PyTypeObject *subtype) self->ob_dict = nullptr; self->weakreflist = nullptr; self->d = d; + PyObject_GC_Track(reinterpret_cast<PyObject *>(self)); return reinterpret_cast<PyObject *>(self); } PyObject *SbkObjectTpNew(PyTypeObject *subtype, PyObject *, PyObject *) { SbkObject *self = PyObject_GC_New(SbkObject, subtype); - PyObject *res = _setupNew(self, subtype); - PyObject_GC_Track(reinterpret_cast<PyObject *>(self)); - return res; + return _setupNew(self, subtype); } PyObject *SbkQAppTpNew(PyTypeObject *subtype, PyObject *, PyObject *) { - // PYSIDE-571: - // For qApp, we need to create a singleton Python object. - // We cannot track this with the GC, because it is a static variable! - - // Python 2 has a weird handling of flags in derived classes that Python 3 - // does not have. Observed with bug_307.py. - // But it could theoretically also happen with Python3. - // Therefore we enforce that there is no GC flag, ever! - - // PYSIDE-560: - // We avoid to use this in Python 3, because we have a hard time to get - // write access to these flags -#ifndef IS_PY3K - if (PyType_HasFeature(subtype, Py_TPFLAGS_HAVE_GC)) { - subtype->tp_flags &= ~Py_TPFLAGS_HAVE_GC; - subtype->tp_free = PyObject_Del; - } -#endif auto self = reinterpret_cast<SbkObject *>(MakeQAppWrapper(subtype)); - return self == nullptr ? nullptr : _setupNew(self, subtype); + if (self == nullptr) + return nullptr; + auto ret = _setupNew(self, subtype); + auto priv = self->d; + priv->isQAppSingleton = 1; + return ret; } PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *) @@ -854,7 +895,7 @@ PyObject *FallbackRichCompare(PyObject *self, PyObject *other, int op) opstrings[op], self->ob_type->tp_name, other->ob_type->tp_name); - return NULL; + return nullptr; } Py_INCREF(res); return res; @@ -1266,6 +1307,12 @@ bool wasCreatedByPython(SbkObject *pyObj) void callCppDestructors(SbkObject *pyObj) { + auto priv = pyObj->d; + if (priv->isQAppSingleton && DestroyQApplication) { + // PYSIDE-1470: Allow to destroy the application from Shiboken. + DestroyQApplication(); + return; + } PyTypeObject *type = Py_TYPE(pyObj); SbkObjectTypePrivate *sotp = PepType_SOTP(type); if (sotp->is_multicpp) { @@ -1278,18 +1325,19 @@ void callCppDestructors(SbkObject *pyObj) sotp->cpp_dtor(pyObj->d->cptr[0]); } + if (priv->validCppObject && priv->containsCppWrapper) { + BindingManager::instance().releaseWrapper(pyObj); + } + /* invalidate needs to be called before deleting pointer array because it needs to delete entries for them from the BindingManager hash table; also release wrapper explicitly if object contains C++ wrapper because invalidate doesn't */ invalidate(pyObj); - if (pyObj->d->validCppObject && pyObj->d->containsCppWrapper) { - BindingManager::instance().releaseWrapper(pyObj); - } - delete[] pyObj->d->cptr; - pyObj->d->cptr = nullptr; - pyObj->d->validCppObject = false; + delete[] priv->cptr; + priv->cptr = nullptr; + priv->validCppObject = false; } bool hasOwnership(SbkObject *pyObj) diff --git a/sources/shiboken2/libshiboken/basewrapper.h b/sources/shiboken2/libshiboken/basewrapper.h index 204c4c1c3..2f0c22e9f 100644 --- a/sources/shiboken2/libshiboken/basewrapper.h +++ b/sources/shiboken2/libshiboken/basewrapper.h @@ -93,9 +93,10 @@ typedef void (*ObjectDestructor)(void *); typedef void (*SubTypeInitHook)(SbkObjectType *, PyObject *, PyObject *); -// PYSIDE-1019: Set the function to select the current feature. +/// PYSIDE-1019: Set the function to select the current feature. +/// Return value is the previous content. typedef PyObject *(*SelectableFeatureHook)(PyTypeObject *); -LIBSHIBOKEN_API void initSelectableFeature(SelectableFeatureHook func); +LIBSHIBOKEN_API SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func); // PYSIDE-1019: Get access to PySide reserved bits. LIBSHIBOKEN_API int SbkObjectType_GetReserved(PyTypeObject *type); @@ -105,6 +106,9 @@ LIBSHIBOKEN_API void SbkObjectType_SetReserved(PyTypeObject *type, int value); LIBSHIBOKEN_API const char **SbkObjectType_GetPropertyStrings(PyTypeObject *type); LIBSHIBOKEN_API void SbkObjectType_SetPropertyStrings(PyTypeObject *type, const char **strings); +/// PYSIDE-1470: Set the function to kill a Q*Application. +typedef void(*DestroyQAppHook)(); +LIBSHIBOKEN_API void setDestroyQApplication(DestroyQAppHook func); extern LIBSHIBOKEN_API PyTypeObject *SbkObjectType_TypeF(void); extern LIBSHIBOKEN_API SbkObjectType *SbkObject_TypeF(void); @@ -118,8 +122,12 @@ struct LIBSHIBOKEN_API SbkObjectType }; LIBSHIBOKEN_API PyObject *SbkObjectTpNew(PyTypeObject *subtype, PyObject *, PyObject *); -// the special case of a switchable singleton -LIBSHIBOKEN_API PyObject *SbkQAppTpNew(PyTypeObject *subtype, PyObject *args, PyObject *kwds); + +/// The special case of a switchable singleton Q*Application. +LIBSHIBOKEN_API PyObject *SbkQAppTpNew(PyTypeObject *subtype, PyObject *, PyObject *); + +/// Create a new Q*Application wrapper and monitor it. +LIBSHIBOKEN_API PyObject *MakeQAppWrapper(PyTypeObject *type); /** * PYSIDE-832: Use object_dealloc instead of nullptr. diff --git a/sources/shiboken2/libshiboken/basewrapper_p.h b/sources/shiboken2/libshiboken/basewrapper_p.h index 64f7941b7..60fba13c5 100644 --- a/sources/shiboken2/libshiboken/basewrapper_p.h +++ b/sources/shiboken2/libshiboken/basewrapper_p.h @@ -97,6 +97,9 @@ struct SbkObjectPrivate unsigned int validCppObject : 1; /// Marked as true when the object constructor was called unsigned int cppObjectCreated : 1; + /// PYSIDE-1470: Marked as true if this is the Q*Application singleton. + /// This bit allows app deletion from shiboken?.delete() . + unsigned int isQAppSingleton : 1; /// Information about the object parents and children, may be null. Shiboken::ParentInfo *parentInfo; /// Manage reference count of objects that are referred to but not owned from. diff --git a/sources/shiboken2/libshiboken/embed/signature_bootstrap.py b/sources/shiboken2/libshiboken/embed/signature_bootstrap.py index b7d9d2793..dd8df2b63 100644 --- a/sources/shiboken2/libshiboken/embed/signature_bootstrap.py +++ b/sources/shiboken2/libshiboken/embed/signature_bootstrap.py @@ -110,8 +110,11 @@ def bootstrap(): rp = os.path.realpath(os.path.dirname(root.__file__)) # This can be the shiboken2 directory or the binary module, so search. look_for = os.path.join("files.dir", "shibokensupport", "signature", "loader.py") - while len(rp) > 3 and not os.path.exists(os.path.join(rp, look_for)): - rp = os.path.abspath(os.path.join(rp, "..")) + while not os.path.exists(os.path.join(rp, look_for)): + dir = os.path.dirname(rp) + if dir == rp: # Hit root, '/', 'C:\', '\\server\share' + break + rp = dir # Here we decide if we work embedded or not. embedding_var = "pyside_uses_embedding" diff --git a/sources/shiboken2/libshiboken/qapp_macro.cpp b/sources/shiboken2/libshiboken/qapp_macro.cpp deleted file mode 100644 index 3ef3a51c6..000000000 --- a/sources/shiboken2/libshiboken/qapp_macro.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2017 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of Qt for Python. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "basewrapper.h" -#include "autodecref.h" - -extern "C" -{ - -#include "qapp_macro.h" - -//////////////////////////////////////////////////////////////////////////// -// -// Support for the qApp macro. -// -// qApp is a macro in Qt5. In Python, we simulate that a little by a -// variable that monitors Q*Application.instance(). -// This variable is also able to destroy the app by qApp.shutdown(). -// - -static PyObject *qApp_var = nullptr; -static PyObject *qApp_content = nullptr; - -static PyObject * -monitor_qApp_var(PyObject *qApp) -{ - static bool init_done; - static PyObject *builtins = PyEval_GetBuiltins(); - - if (!init_done) { - qApp_var = Py_BuildValue("s", "qApp"); - if (qApp_var == nullptr) - return nullptr; - // This is a borrowed reference - Py_INCREF(builtins); - init_done = true; - } - - if (PyDict_SetItem(builtins, qApp_var, qApp) < 0) - return nullptr; - qApp_content = qApp; - Py_INCREF(qApp); - return qApp; -} - -PyObject * -MakeQAppWrapper(PyTypeObject *type) -{ - if (type == nullptr) - type = Py_TYPE(Py_None); - if (!(type == Py_TYPE(Py_None) || Py_TYPE(qApp_content) == Py_TYPE(Py_None))) { - const char *res_name = PepType_GetNameStr(Py_TYPE(qApp_content)); - const char *type_name = PepType_GetNameStr(type); - PyErr_Format(PyExc_RuntimeError, "Please destroy the %s singleton before" - " creating a new %s instance.", res_name, type_name); - return nullptr; - } - PyObject *self = type != Py_TYPE(Py_None) ? PyObject_New(PyObject, type) : Py_None; - return monitor_qApp_var(self); -} - -} //extern "C" - -// end of module diff --git a/sources/shiboken2/libshiboken/qapp_macro.h b/sources/shiboken2/libshiboken/qapp_macro.h deleted file mode 100644 index 9abd17c17..000000000 --- a/sources/shiboken2/libshiboken/qapp_macro.h +++ /dev/null @@ -1,52 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2017 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of Qt for Python. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QAPP_MACRO_H -#define QAPP_MACRO_H - -#include "sbkpython.h" - -extern "C" -{ - -LIBSHIBOKEN_API PyObject *MakeQAppWrapper(PyTypeObject *type); - -} // extern "C" - -#endif // QAPP_MACRO_H diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings.cpp b/sources/shiboken2/libshiboken/sbkstaticstrings.cpp index 2c1a9b891..58d32f217 100644 --- a/sources/shiboken2/libshiboken/sbkstaticstrings.cpp +++ b/sources/shiboken2/libshiboken/sbkstaticstrings.cpp @@ -58,7 +58,10 @@ STATIC_STRING_IMPL(fset, "fset") STATIC_STRING_IMPL(loads, "loads") STATIC_STRING_IMPL(multi, "multi") STATIC_STRING_IMPL(name, "name") +STATIC_STRING_IMPL(qApp, "qApp") STATIC_STRING_IMPL(result, "result") +STATIC_STRING_IMPL(select_id, "select_id") +STATIC_STRING_IMPL(underscore, "_") STATIC_STRING_IMPL(value, "value") STATIC_STRING_IMPL(values, "values") diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings.h b/sources/shiboken2/libshiboken/sbkstaticstrings.h index 0dd533e32..4aaef814f 100644 --- a/sources/shiboken2/libshiboken/sbkstaticstrings.h +++ b/sources/shiboken2/libshiboken/sbkstaticstrings.h @@ -58,6 +58,8 @@ LIBSHIBOKEN_API PyObject *loads(); LIBSHIBOKEN_API PyObject *multi(); LIBSHIBOKEN_API PyObject *name(); LIBSHIBOKEN_API PyObject *result(); +LIBSHIBOKEN_API PyObject *select_id(); +LIBSHIBOKEN_API PyObject *underscore(); LIBSHIBOKEN_API PyObject *value(); LIBSHIBOKEN_API PyObject *values(); } // namespace PyName @@ -69,6 +71,7 @@ LIBSHIBOKEN_API PyObject *dict(); LIBSHIBOKEN_API PyObject *doc(); LIBSHIBOKEN_API PyObject *ecf(); LIBSHIBOKEN_API PyObject *file(); +LIBSHIBOKEN_API PyObject *func(); LIBSHIBOKEN_API PyObject *get(); LIBSHIBOKEN_API PyObject *members(); LIBSHIBOKEN_API PyObject *module(); diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings_p.h b/sources/shiboken2/libshiboken/sbkstaticstrings_p.h index c33fa0299..419eeeb10 100644 --- a/sources/shiboken2/libshiboken/sbkstaticstrings_p.h +++ b/sources/shiboken2/libshiboken/sbkstaticstrings_p.h @@ -51,6 +51,7 @@ PyObject *marshal(); PyObject *method(); PyObject *mro(); PyObject *overload(); +PyObject *qApp(); PyObject *staticmethod(); } // namespace PyName namespace PyMagicName diff --git a/sources/shiboken2/libshiboken/signature/signature.cpp b/sources/shiboken2/libshiboken/signature/signature.cpp index 1f36a09f3..4c251af5b 100644 --- a/sources/shiboken2/libshiboken/signature/signature.cpp +++ b/sources/shiboken2/libshiboken/signature/signature.cpp @@ -525,13 +525,11 @@ void SetError_Argument(PyObject *args, const char *func_name, PyObject *info) init_module_2(); // PYSIDE-1305: Handle errors set by fillQtProperties. - PyObject *err_val{}; if (PyErr_Occurred()) { - PyObject *e, *t; - PyErr_Fetch(&e, &err_val, &t); - info = err_val; - Py_XDECREF(&e); - Py_XDECREF(&t); + PyObject *e, *v, *t; + // Note: These references are all borrowed. + PyErr_Fetch(&e, &v, &t); + info = v; } // PYSIDE-1019: Modify the function name expression according to feature. AutoDecRef new_func_name(adjustFuncName(func_name)); @@ -552,7 +550,6 @@ void SetError_Argument(PyObject *args, const char *func_name, PyObject *info) PyErr_Print(); Py_FatalError("unexpected failure in seterror_argument"); } - Py_XDECREF(err_val); PyErr_SetObject(err, msg); } diff --git a/tools/debug_renamer.py b/tools/debug_renamer.py new file mode 100644 index 000000000..da5beb127 --- /dev/null +++ b/tools/debug_renamer.py @@ -0,0 +1,122 @@ +############################################################################# +## +## Copyright (C) 2020 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the test suite of Qt for Python. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +debug_renamer.py +================ + +This script renames object addresses in debug protocols to useful names. +Comparing output will produce minimal deltas. + + +Problem: +-------- + +In the debugging output of PYSIDE-79, we want to study different output +before and after applying some change to the implementation. + +We have support from the modified Python interpreter that creates full +traces of every object creation and increment/decrement of refcounts. + +The comparison between "before" and "after" gets complicated because +the addresses of objects do not compare well. + + +Input format: +------------- +The Python output lines are of this format: + +mode filename:lineno funcname object_id typename object_refcount + +Mode can be "INC", "DEC", "XINC", XDEC", "NEW, "NEWV". + +On "NEW" or "NEWV", an object is created and the refcount is always 1. +On "DEC" or "XDEC", when refcount is 0, the object is deleted. + + +Operation +--------- + +The script reads from <stdin> until EOF. It produces output where the +object_id field is removed and some text is combined with object_typename +to produce a unique object name. + + +Example +------- + +You can create reference debugging output by using the modified interpreter at + + https://github.com/ctismer/cpython/tree/3.9-refdebug + +and pipe the error output through this script. +This is work in flux that might change quite often. + + +To Do List +---------- + +The script should be re-worked to be more flexible, without relying on +the number of coulumns but with some intelligent guessing. + +Names of objects which are already deleted should be monitored and +not by chance be re-used. +""" + +import sys +from collections import OrderedDict + + +def make_name(type_name, name_pos): + """ + Build a name by using uppercase letters and numbers + """ + if name_pos < 26: + name = chr(ord("A") + name_pos) + return f"{type_name}_{name}" + return f"{type_name}_{str(name_pos)}" + + +mode_tokens = "NEW NEWV INC DEC XINC XDEC".split() +known_types = {} + +while 1: + line = sys.stdin.readline() + if not line: + break + fields = line.split() + if len(fields) != 6 or fields[0] not in mode_tokens: + print(line.rstrip()) + continue + mode, fname_lno, funcname, object_id, typename, refcount = fields + if typename not in known_types: + known_types[typename] = OrderedDict() + obj_store = known_types[typename] + if object_id not in obj_store: + obj_store[object_id] = make_name(typename, len(obj_store)) + print(f"{mode} {fname_lno} {funcname} {obj_store[object_id]} {refcount}") diff --git a/tools/leak_finder.py b/tools/leak_finder.py new file mode 100644 index 000000000..5b5102887 --- /dev/null +++ b/tools/leak_finder.py @@ -0,0 +1,196 @@ +############################################################################# +## +## Copyright (C) 2020 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the test suite of Qt for Python. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +leak_finder.py +============== + +This script finds memory leaks in Python. + +Usage: +------ + +Place one or more lines which should be tested for leaks in a loop: + + from leak_finder import LeakFinder + ... + lf = LeakFinder() + for i in range(1000): + leaking_statement() + lf.find_leak() + + +Theory +------ + +How to find a leak? + +We repeatedly perform an action and observe if that has an unexpected +side effect. There are typically two observations: + +* one object is growing its refcount (a pseudo-leak) +* we get many new objects of one type (a true leak) + +A difficulty in trying to get leak info is avoiding side effects +of the measurement. Early attempts with lists of refcounts were +unsuccessful. Using array.array for counting refcounts avoids that. + + +Algorithm +--------- +We record a snapshot of all objects in a list and a parallel array +of refcounts. + +Then we do some computation and do the same snapshot again. + +The structure of a list of all objects is extending at the front for +some reason. That makes the captured structures easy to compare. +We reverse that list and array and have for the objects: + + len(all2) >= len(all1) + + all1[idx] == all2[idx] for idx in range(len(all1)) + +When taking the second snapshot, the objects still have references from +the first snapshot. +For objects with no effect, the following relation is true: + + refs1[idx] == refs2[idx] - 1 for idx in range(len(all1)) + +All other objects are potential pseudo-leaks, because they waste +references but no objects in the first place. + +Then we look at the newly created objects: +These objects are real leaks if their number is growing with the probe +size. For analysis, the number of new objects per type is counted. +""" + +import sys +import gc +import array +import unittest + +# this comes from Python, too +from test import support + +try: + sys.getobjects + have_debug = True +except AttributeError: + have_debug = False + + +class LeakFinder(object): + def __init__(self): + self.all, self.refs = self._make_snapshot() + + @staticmethod + def _make_snapshot(): + gc.collect() + # get all objects + all = sys.getobjects(0) + # get an array with the refcounts + g = sys.getrefcount + refs = array.array("l", (g(obj) for obj in all)) + # the lists have the same endind. Make comparison easier. + all.reverse() + refs.reverse() + return all, refs + + @staticmethod + def _short_repr(x, limit=76): + s = repr(x) + if len(s) > limit: + s = s[:limit] + "..." + return s + + def find_leak(self): + all1 = self.all + refs1 = self.refs + del self.all, self.refs + all2, refs2 = self._make_snapshot() + common = len(all1) + del all1 + + srepr = self._short_repr + # look into existing objects for increased refcounts + first = True + for idx in range(common): + ref = refs2[idx] - refs1[idx] - 1 + if abs(ref) <= 10: + continue + obj = all2[idx] + if first: + print() + first = False + print(f"Fake Leak ref={ref} obj={srepr(obj)}") + + # look at the extra objects by type size + types = {} + for idx in range(common, len(all2)): + obj = all2[idx] + typ = type(obj) + if typ not in types: + types[typ] = [] + types[typ].append(obj) + first = True + for typ in types: + oblis = types[typ] + ref = len(oblis) + if ref <= 10: + continue + try: + oblis.sort() + except TypeError: + pass + if first: + print() + first = False + left, mid, right = oblis[0], oblis[ref // 2], oblis[-1] + print(f"True Leak ref={ref} typ={typ} left={left} mid={mid} right={right}") + + +class TestDemo(unittest.TestCase): + + @unittest.skipUnless(have_debug, 'You need a debug build with "--with-trace-refs"') + def test_demo(self): + # create a pseudo leak and a true leak + fake_leak_obj = [] + true_leak_obj = [] + lf = LeakFinder() + refs_before = sys.gettotalrefcount() + for idx in range(100): + fake_leak_obj.append("same string") + true_leak_obj.append(idx + 1000) # avoiding cached low numbers + refs_after = sys.gettotalrefcount() + lf.find_leak() + self.assertNotAlmostEqual(refs_after - refs_before, 0, delta=10) + + +if __name__ == "__main__": + unittest.main() |