shiboken: Update sbkstring to use PyUnicode_GetLength

The unicode implementation has changed very much.
PyUnicode_GET_SIZE has been used all the time, although this
macro is already a quite expensive real function since Python 3.3 .

The function is deprecated, and the macro PyUnicode_GET_LENGTH
should be used, instead.

This is relevant for cleaning things up, because in the course of
fixing PYSIDE-813 we work with debug Python, which then complains
that Python memory is used without holding the GIL.

The usage of the right implementation was ensured by a function
PepUnicode_GetLength and removing all traces of the former version.

Task-number: PYSIDE-813
Change-Id: I62e94e10e14975dac3dad0ed1fffec8a1b54a0d5
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

7 files changed, 39 insertions, 14 deletions

diff --git a/sources/pyside2/PySide2/glue/qtcore.cpp b/sources/pyside2/PySide2/glue/qtcore.cpp
index 60ddf675c..169b89cae 100644
--- a/sources/pyside2/PySide2/glue/qtcore.cpp
+++ b/sources/pyside2/PySide2/glue/qtcore.cpp
@@ -1685,7 +1685,7 @@ Py_UNICODE *unicode = PyUnicode_AS_UNICODE(%in);
 // cast as Py_UNICODE can be a different type
 %out = QString::fromUcs4((const uint *)unicode);
 # else
-%out = QString::fromUtf16((const ushort *)unicode, PyUnicode_GET_SIZE(%in));
+%out = QString::fromUtf16((const ushort *)unicode, PepUnicode_GetLength(%in));
 # endif
 #else
 wchar_t *temp = PyUnicode_AsWideCharString(%in, NULL);
diff --git a/sources/shiboken2/libshiboken/bufferprocs_py37.cpp b/sources/shiboken2/libshiboken/bufferprocs_py37.cpp
index ddb07390e..da6bb00a3 100644
--- a/sources/shiboken2/libshiboken/bufferprocs_py37.cpp
+++ b/sources/shiboken2/libshiboken/bufferprocs_py37.cpp
@@ -47,7 +47,7 @@
 
 #ifdef Py_LIMITED_API
 
-#include "pep384impl.h"
+#include "sbkpython.h"
 /* Buffer C-API for Python 3.0 */
 
 int
diff --git a/sources/shiboken2/libshiboken/pep384impl.cpp b/sources/shiboken2/libshiboken/pep384impl.cpp
index 622efd201..9939bb410 100644
--- a/sources/shiboken2/libshiboken/pep384impl.cpp
+++ b/sources/shiboken2/libshiboken/pep384impl.cpp
@@ -37,7 +37,7 @@
 **
 ****************************************************************************/
 
-#include "pep384impl.h"
+#include "sbkpython.h"
 #include "autodecref.h"
 #include "sbkstaticstrings.h"
 #include "sbkstaticstrings_p.h"
diff --git a/sources/shiboken2/libshiboken/pep384impl.h b/sources/shiboken2/libshiboken/pep384impl.h
index 6d0fa2450..c18f00804 100644
--- a/sources/shiboken2/libshiboken/pep384impl.h
+++ b/sources/shiboken2/libshiboken/pep384impl.h
@@ -40,8 +40,6 @@
 #ifndef PEP384IMPL_H
 #define PEP384IMPL_H
 
-#include "sbkpython.h"
-
 extern "C"
 {
 
@@ -204,17 +202,40 @@ LIBSHIBOKEN_API int Pep_GetVerboseFlag(void);
  * RESOLVED: unicodeobject.h
  *
  */
-#ifdef Py_LIMITED_API
-
-LIBSHIBOKEN_API char *_PepUnicode_AsString(PyObject *);
 
+///////////////////////////////////////////////////////////////////////
+//
+// PYSIDE-813: About The Length Of Unicode Objects
+// -----------------------------------------------
+//
+// In Python 2 and before Python 3.3, the macro PyUnicode_GET_SIZE
+// worked fine and really like a macro.
+//
+// Meanwhile, the unicode objects have changed their layout very much,
+// and the former cheap macro call has become a real function call
+// that converts objects and needs PyMemory.
+// PyUnicode_GET_SIZE was retained for compatibility reasons.
+//
+// That is not only inefficient, but also requires the GIL!
+// This problem was visible by debug Python and qdatastream_test.py .
+// It was found while fixing the refcount problem of PYSIDE-813 which
+// needed a debug Python.
+//
+
+// PyUnicode_GetSize is deprecated in favor of PyUnicode_GetLength.
+// We undefine the PyUnicode_GET_SIZE macro, to be sure that it is not used
+// by accident. Only PepUnicode_GetLength should be used.
+#undef PyUnicode_GET_SIZE
 #if PY_VERSION_HEX < 0x03000000
-#define PyUnicode_GET_SIZE(op)      PyUnicode_GetSize((PyObject *)(op))
+#define PepUnicode_GetLength(op)    PyUnicode_GetSize((PyObject *)(op))
 #else
-// PyUnicode_GetSize is deprecated in favor of PyUnicode_GetLength
-#define PyUnicode_GET_SIZE(op)      PyUnicode_GetLength((PyObject *)(op))
+#define PepUnicode_GetLength(op)    PyUnicode_GetLength((PyObject *)(op))
 #endif
 
+#ifdef Py_LIMITED_API
+
+LIBSHIBOKEN_API char *_PepUnicode_AsString(PyObject *);
+
 #else
 #define _PepUnicode_AsString     PyUnicode_AsUTF8
 #endif
diff --git a/sources/shiboken2/libshiboken/pep384impl_doc.rst b/sources/shiboken2/libshiboken/pep384impl_doc.rst
index 2f3b7ea97..ac4b053a7 100644
--- a/sources/shiboken2/libshiboken/pep384impl_doc.rst
+++ b/sources/shiboken2/libshiboken/pep384impl_doc.rst
@@ -70,8 +70,10 @@ supported. We redefined it as macro ``Py_VerboseFlag`` which calls ``Pep_Verbose
 unicodeobject.h
 ---------------
 
-The macro ``PyUnicode_GET_SIZE`` was redefined to call into ``PyUnicode_GetSize``
-for Python 2, and ``PyUnicode_GetLength`` for Python 3.
+The macro ``PyUnicode_GET_SIZE`` was removed and replaced by ``PepUnicode_GetLength``
+which evaluates to ``PyUnicode_GetSize`` for Python 2 and ``PyUnicode_GetLength`` for Python 3.
+Since Python 3.3, ``PyUnicode_GetSize`` would have the bad side effect of requiring the GIL!
+
 Function ``_PyUnicode_AsString`` is unavailable and was replaced by a macro
 that calls ``_PepUnicode_AsString``. The implementation was a bit involved,
 and it would be better to change the code and replace this function.
diff --git a/sources/shiboken2/libshiboken/sbkpython.h b/sources/shiboken2/libshiboken/sbkpython.h
index f06b0b19e..9dd1e712e 100644
--- a/sources/shiboken2/libshiboken/sbkpython.h
+++ b/sources/shiboken2/libshiboken/sbkpython.h
@@ -72,6 +72,7 @@ extern "C" {
 // Now we have the usual variables from Python.h .
 #  include "python25compat.h"
 #  include "shibokenmacros.h"
+// "pep384impl.h" may nowhere be included but in this file.
 #  include "pep384impl.h"
 #  include "typespec.h"
 #  pragma pop_macro("slots")
@@ -98,6 +99,7 @@ extern "C" {
 // Now we have the usual variables from Python.h .
 #  include "python25compat.h"
 #  include "shibokenmacros.h"
+// "pep384impl.h" may nowhere be included but in this file.
 #  include "pep384impl.h"
 #  include "typespec.h"
 #endif
diff --git a/sources/shiboken2/libshiboken/sbkstring.cpp b/sources/shiboken2/libshiboken/sbkstring.cpp
index 4a441222f..38bb105d0 100644
--- a/sources/shiboken2/libshiboken/sbkstring.cpp
+++ b/sources/shiboken2/libshiboken/sbkstring.cpp
@@ -202,7 +202,7 @@ Py_ssize_t len(PyObject *str)
         return 0;
 
     if (PyUnicode_Check(str))
-        return PyUnicode_GET_SIZE(str);
+        return PepUnicode_GetLength(str);
 
     if (PyBytes_Check(str))
         return PyBytes_GET_SIZE(str);