Improve the NumPy Support by iterables

Working example, by overriding cppgenerator:

>>> from PySide2 import *
>>> QtCore.QUrl.fromStringList(("asd", "def"))
[PySide2.QtCore.QUrl('asd'), PySide2.QtCore.QUrl('def')]
>>> def func(lis):
...   for thing in lis:
...      yield thing
...
>>> QtCore.QUrl.fromStringList(func(["asd", "def"]))
[PySide2.QtCore.QUrl('asd'), PySide2.QtCore.QUrl('def')]

Also working, by overriding shibokengenerator

>>> QtGui.QMatrix4x4(func(range(16)))

And all other QMatrix sizes as well:

>>> QtGui.QMatrix2x2(func(range(4)))
>>> QtGui.QMatrix2x3(func(range(6)))

The PySequence cases seem to be quite completely covered.
Supporting lists and QVector is not yet clear and needs
more research.

Note.. QtOpenGLFunctions is not tested at all and nothing works
    on macOS, segfault. Ignored for now!

A simple numpy test shows how versatile this solution is.
We now need to improve signatures and error messages
to optimize the experience.

Task-number: PYSIDE-795
Change-Id: I195cd46cf47c2eb83276fe48fce8e6070cf30fda
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

14 files changed, 158 insertions, 27 deletions

diff --git a/sources/pyside2/PySide2/glue/qtgui.cpp b/sources/pyside2/PySide2/glue/qtgui.cpp
index 5be8cc287..d2480e99e 100644
--- a/sources/pyside2/PySide2/glue/qtgui.cpp
+++ b/sources/pyside2/PySide2/glue/qtgui.cpp
@@ -455,10 +455,12 @@ QPoint p(%CPPSELF.%FUNCTION_NAME(%1));
 // @snippet qmatrix-map-point
 
 // @snippet qmatrix4x4
-if (PySequence_Size(%PYARG_1) == 16) {
+// PYSIDE-795: All PySequences can be made iterable with PySequence_Fast.
+Shiboken::AutoDecRef seq(PySequence_Fast(%PYARG_1, "Can't turn into sequence"));
+if (PySequence_Size(seq) == 16) {
     float values[16];
     for (int i=0; i < 16; ++i) {
-        PyObject *pv = PySequence_Fast_GET_ITEM(%PYARG_1, i);
+        PyObject *pv = PySequence_Fast_GET_ITEM(seq.object(), i);
         values[i] = PyFloat_AsDouble(pv);
     }
 
diff --git a/sources/pyside2/PySide2/templates/core_common.xml b/sources/pyside2/PySide2/templates/core_common.xml
index 4b74c6b55..8147b39e8 100644
--- a/sources/pyside2/PySide2/templates/core_common.xml
+++ b/sources/pyside2/PySide2/templates/core_common.xml
@@ -339,9 +339,16 @@
     </template>
 
     <template name="pyseq_to_cpplist_conversion">
-    for (int i = 0, size = PySequence_Size(%in); i &lt; size; i++) {
-
-        Shiboken::AutoDecRef pyItem(PySequence_GetItem(%in, i));
+    // PYSIDE-795: Turn all sequences into iterables.
+    Shiboken::AutoDecRef it(PyObject_GetIter(%in));
+    PyObject *(*iternext)(PyObject *) = *Py_TYPE(it)->tp_iternext;
+    for (;;) {
+        Shiboken::AutoDecRef pyItem(iternext(it));
+        if (pyItem.isNull()) {
+            if (PyErr_Occurred() &amp;&amp; PyErr_ExceptionMatches(PyExc_StopIteration))
+                PyErr_Clear();
+            break;
+        }
         %OUTTYPE_0 cppItem = %CONVERTTOCPP[%OUTTYPE_0](pyItem);
         %out &lt;&lt; cppItem;
     }
@@ -358,10 +365,20 @@
     </template>
 
     <template name="pyseq_to_cppvector_conversion">
-    int vectorSize = PySequence_Size(%in);
-    %out.reserve(vectorSize);
-    for (int idx = 0; idx &lt; vectorSize; ++idx) {
-        Shiboken::AutoDecRef pyItem(PySequence_GetItem(%in, idx));
+    // PYSIDE-795: Turn all sequences into iterables.
+    if (PySequence_Check(%in)) {
+        int vectorSize = PySequence_Size(%in);
+        %out.reserve(vectorSize);
+    }
+    Shiboken::AutoDecRef it(PyObject_GetIter(%in));
+    PyObject *(*iternext)(PyObject *) = *Py_TYPE(it)->tp_iternext;
+    for (;;) {
+        Shiboken::AutoDecRef pyItem(iternext(it));
+        if (pyItem.isNull()) {
+            if (PyErr_Occurred() &amp;&amp; PyErr_ExceptionMatches(PyExc_StopIteration))
+                PyErr_Clear();
+            break;
+        }
         %OUTTYPE_0 cppItem = %CONVERTTOCPP[%OUTTYPE_0](pyItem);
         %out.push_back(cppItem);
     }
diff --git a/sources/pyside2/PySide2/templates/gui_common.xml b/sources/pyside2/PySide2/templates/gui_common.xml
index f3e772a8c..96b4906ef 100644
--- a/sources/pyside2/PySide2/templates/gui_common.xml
+++ b/sources/pyside2/PySide2/templates/gui_common.xml
@@ -239,8 +239,10 @@
     </template>
 
     <template name="matrix_constructor">
-        if (PySequence_Size(%PYARG_1) == %SIZE) {
-            Shiboken::AutoDecRef fast(PySequence_Fast(%PYARG_1,
+        // PYSIDE-795: All PySequences can be made iterable with PySequence_Fast.
+        Shiboken::AutoDecRef seq(PySequence_Fast(%PYARG_1, "Can't turn into sequence"));
+        if (PySequence_Size(seq) == %SIZE) {
+            Shiboken::AutoDecRef fast(PySequence_Fast(seq,
                 "Failed to parse sequence on %TYPE constructor."));
             float values[%SIZE];
             for(int i=0; i &lt; %SIZE; i++) {
diff --git a/sources/pyside2/tests/pysidetest/CMakeLists.txt b/sources/pyside2/tests/pysidetest/CMakeLists.txt
index 3c993cf4e..f81de55f7 100644
--- a/sources/pyside2/tests/pysidetest/CMakeLists.txt
+++ b/sources/pyside2/tests/pysidetest/CMakeLists.txt
@@ -124,21 +124,22 @@ PYSIDE_TEST(decoratedslot_test.py)
 if (Qt5Core_VERSION VERSION_GREATER 5.7.0)
     PYSIDE_TEST(delegatecreateseditor_test.py)
 endif()
+PYSIDE_TEST(all_modules_load_test.py)
+PYSIDE_TEST(bug_1016.py)
+PYSIDE_TEST(embedding_test.py)
 PYSIDE_TEST(enum_test.py)
 PYSIDE_TEST(homonymoussignalandmethod_test.py)
+PYSIDE_TEST(iterable_test.py)
 PYSIDE_TEST(list_signal_test.py)
+PYSIDE_TEST(mixin_signal_slots_test.py)
 PYSIDE_TEST(modelview_test.py)
 PYSIDE_TEST(new_inherited_functions_test.py)
 PYSIDE_TEST(notify_id.py)
+PYSIDE_TEST(qapp_like_a_macro_test.py)
 PYSIDE_TEST(qvariant_test.py)
+PYSIDE_TEST(signal_slot_warning.py)
 PYSIDE_TEST(signalandnamespace_test.py)
-PYSIDE_TEST(signalwithdefaultvalue_test.py)
 PYSIDE_TEST(signalemissionfrompython_test.py)
-PYSIDE_TEST(version_test.py)
+PYSIDE_TEST(signalwithdefaultvalue_test.py)
 PYSIDE_TEST(typedef_signal_test.py)
-PYSIDE_TEST(bug_1016.py)
-PYSIDE_TEST(mixin_signal_slots_test.py)
-PYSIDE_TEST(signal_slot_warning.py)
-PYSIDE_TEST(all_modules_load_test.py)
-PYSIDE_TEST(qapp_like_a_macro_test.py)
-PYSIDE_TEST(embedding_test.py)
+PYSIDE_TEST(version_test.py)
diff --git a/sources/pyside2/tests/pysidetest/iterable_test.py b/sources/pyside2/tests/pysidetest/iterable_test.py
new file mode 100644
index 000000000..2f488875e
--- /dev/null
+++ b/sources/pyside2/tests/pysidetest/iterable_test.py
@@ -0,0 +1,86 @@
+#############################################################################
+##
+## Copyright (C) 2019 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the test suite of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+"""
+iterable_test.py
+
+This test checks that the Iterable protocol is implemented correctly.
+"""
+
+import os
+import sys
+import unittest
+import PySide2
+from PySide2 import QtCore, QtGui
+
+try:
+    import numpy as np
+    have_numpy = True
+except ImportError:
+    have_numpy = False
+
+class PySequenceTest(unittest.TestCase):
+
+    def test_iterable(self):
+        def gen(lis):
+            for item in lis:
+                if item == "crash":
+                    raise IndexError
+                yield item
+        # testing "pyseq_to_cpplist_conversion"
+        testfunc = QtCore.QUrl.fromStringList
+        # use a generator (iterable)
+        self.assertEqual(testfunc(gen(["asd", "ghj"])),
+            [PySide2.QtCore.QUrl('asd'), PySide2.QtCore.QUrl('ghj')])
+        # use an iterator
+        self.assertEqual(testfunc(iter(["asd", "ghj"])),
+            [PySide2.QtCore.QUrl('asd'), PySide2.QtCore.QUrl('ghj')])
+        self.assertRaises(IndexError, testfunc, gen(["asd", "crash", "ghj"]))
+        # testing QMatrix4x4
+        testfunc = QtGui.QMatrix4x4
+        self.assertEqual(testfunc(gen(range(16))), testfunc(range(16)))
+        # Note: The errormessage needs to be improved!
+        # We should better get a ValueError
+        self.assertRaises((TypeError, ValueError), testfunc, gen(range(15)))
+        # All other matrix sizes:
+        testfunc = QtGui.QMatrix2x2
+        self.assertEqual(testfunc(gen(range(4))), testfunc(range(4)))
+        testfunc = QtGui.QMatrix2x3
+        self.assertEqual(testfunc(gen(range(6))), testfunc(range(6)))
+
+    @unittest.skipUnless(have_numpy, "requires numpy")
+    def test_iterable_numpy(self):
+        # Demo for numpy: We create a unit matrix.
+        num_mat = np.eye(4)
+        num_mat.shape = 16
+        unit = QtGui.QMatrix4x4(num_mat)
+        self.assertEqual(unit, QtGui.QMatrix4x4())
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
index 547aecc61..146a53d2d 100644
--- a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
+++ b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
@@ -2155,6 +2155,8 @@ void CppGenerator::writeTypeCheck(QTextStream &s, const AbstractMetaType *argTyp
     QString customCheck;
     if (!customType.isEmpty()) {
         AbstractMetaType *metaType;
+        // PYSIDE-795: Note: XML-Overrides are handled in this shibokengenerator function!
+        // This enables iterables for QMatrix4x4 for instance.
         customCheck = guessCPythonCheckFunction(customType, &metaType);
         if (metaType)
             argType = metaType;
@@ -2940,6 +2942,9 @@ void CppGenerator::writePythonToCppConversionFunctions(QTextStream &s,
             typeCheck = QLatin1String("PyType_Check(%in)");
         else if (pyTypeName == QLatin1String("PyObject"))
             typeCheck = QLatin1String("PyObject_TypeCheck(%in, &PyBaseObject_Type)");
+        // PYSIDE-795: We abuse PySequence for iterables
+        else if (pyTypeName == QLatin1String("PySequence"))
+            typeCheck = QLatin1String("Shiboken::String::checkIterable(%in)");
         else if (pyTypeName.startsWith(QLatin1String("Py")))
             typeCheck = pyTypeName + QLatin1String("_Check(%in)");
     }
diff --git a/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp b/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp
index a94d3c8a7..67137369c 100644
--- a/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp
+++ b/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp
@@ -1255,6 +1255,11 @@ QString ShibokenGenerator::cpythonCheckFunction(const TypeEntry *type, bool gene
 QString ShibokenGenerator::guessCPythonCheckFunction(const QString &type, AbstractMetaType **metaType)
 {
     *metaType = nullptr;
+    // PYSIDE-795: We abuse PySequence for iterables.
+    // This part handles the overrides in the XML files.
+    if (type == QLatin1String("PySequence"))
+        return QLatin1String("Shiboken::String::checkIterable");
+
     if (type == QLatin1String("PyTypeObject"))
         return QLatin1String("PyType_Check");
 
diff --git a/sources/shiboken2/libshiboken/pep384impl.cpp b/sources/shiboken2/libshiboken/pep384impl.cpp
index f4e404bed..5729100bf 100644
--- a/sources/shiboken2/libshiboken/pep384impl.cpp
+++ b/sources/shiboken2/libshiboken/pep384impl.cpp
@@ -87,14 +87,15 @@ static PyGetSetDef probe_getseters[] = {
 #define probe_tp_str        make_dummy(2)
 #define probe_tp_traverse   make_dummy(3)
 #define probe_tp_clear      make_dummy(4)
+#define probe_tp_iternext   make_dummy(5)
 #define probe_tp_methods    probe_methoddef
 #define probe_tp_getset     probe_getseters
-#define probe_tp_descr_get  make_dummy(7)
-#define probe_tp_init       make_dummy(8)
-#define probe_tp_alloc      make_dummy(9)
-#define probe_tp_new        make_dummy(10)
-#define probe_tp_free       make_dummy(11)
-#define probe_tp_is_gc      make_dummy(12)
+#define probe_tp_descr_get  make_dummy(8)
+#define probe_tp_init       make_dummy(9)
+#define probe_tp_alloc      make_dummy(10)
+#define probe_tp_new        make_dummy(11)
+#define probe_tp_free       make_dummy(12)
+#define probe_tp_is_gc      make_dummy(13)
 
 #define probe_tp_name       "type.probe"
 #define probe_tp_basicsize  make_dummy_int(42)
@@ -104,6 +105,7 @@ static PyType_Slot typeprobe_slots[] = {
     {Py_tp_str,         probe_tp_str},
     {Py_tp_traverse,    probe_tp_traverse},
     {Py_tp_clear,       probe_tp_clear},
+    {Py_tp_iternext,    probe_tp_iternext},
     {Py_tp_methods,     probe_tp_methods},
     {Py_tp_getset,      probe_tp_getset},
     {Py_tp_descr_get,   probe_tp_descr_get},
@@ -145,6 +147,7 @@ check_PyTypeObject_valid()
         || probe_tp_traverse        != check->tp_traverse
         || probe_tp_clear           != check->tp_clear
         || probe_tp_weakrefoffset   != typetype->tp_weaklistoffset
+        || probe_tp_iternext        != check->tp_iternext
         || probe_tp_methods         != check->tp_methods
         || probe_tp_getset          != check->tp_getset
         || probe_tp_base            != typetype->tp_base
diff --git a/sources/shiboken2/libshiboken/pep384impl.h b/sources/shiboken2/libshiboken/pep384impl.h
index 93f718988..1aa7e6fc0 100644
--- a/sources/shiboken2/libshiboken/pep384impl.h
+++ b/sources/shiboken2/libshiboken/pep384impl.h
@@ -110,7 +110,7 @@ typedef struct _typeobject {
     void *X23; // richcmpfunc tp_richcompare;
     Py_ssize_t tp_weaklistoffset;
     void *X25; // getiterfunc tp_iter;
-    void *X26; // iternextfunc tp_iternext;
+    iternextfunc tp_iternext;
     struct PyMethodDef *tp_methods;
     void *X28; // struct PyMemberDef *tp_members;
     struct PyGetSetDef *tp_getset;
diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings.cpp b/sources/shiboken2/libshiboken/sbkstaticstrings.cpp
index dcd25da16..3727eb494 100644
--- a/sources/shiboken2/libshiboken/sbkstaticstrings.cpp
+++ b/sources/shiboken2/libshiboken/sbkstaticstrings.cpp
@@ -81,6 +81,7 @@ STATIC_STRING_IMPL(code, "__code__")
 STATIC_STRING_IMPL(dictoffset, "__dictoffset__")
 STATIC_STRING_IMPL(func, "__func__")
 STATIC_STRING_IMPL(func_kind, "__func_kind__")
+STATIC_STRING_IMPL(iter, "__iter__")
 STATIC_STRING_IMPL(mro, "__mro__")
 STATIC_STRING_IMPL(new_, "__new__")
 STATIC_STRING_IMPL(objclass, "__objclass__")
diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings_p.h b/sources/shiboken2/libshiboken/sbkstaticstrings_p.h
index bb4cb13c3..42c5585fa 100644
--- a/sources/shiboken2/libshiboken/sbkstaticstrings_p.h
+++ b/sources/shiboken2/libshiboken/sbkstaticstrings_p.h
@@ -61,6 +61,7 @@ PyObject *code();
 PyObject *dictoffset();
 PyObject *func();
 PyObject *func_kind();
+PyObject *iter();
 PyObject *module();
 PyObject *mro();
 PyObject *new_();
diff --git a/sources/shiboken2/libshiboken/sbkstring.cpp b/sources/shiboken2/libshiboken/sbkstring.cpp
index 2e638a413..4a441222f 100644
--- a/sources/shiboken2/libshiboken/sbkstring.cpp
+++ b/sources/shiboken2/libshiboken/sbkstring.cpp
@@ -38,6 +38,7 @@
 ****************************************************************************/
 
 #include "sbkstring.h"
+#include "sbkstaticstrings_p.h"
 #include "autodecref.h"
 
 #include <vector>
@@ -48,6 +49,12 @@ namespace Shiboken
 namespace String
 {
 
+// PYSIDE-795: Redirecting PySequence to Iterable
+bool checkIterable(PyObject *obj)
+{
+    return PyObject_HasAttr(obj, Shiboken::PyMagicName::iter());
+}
+
 bool checkType(PyTypeObject *type)
 {
     return type == &PyUnicode_Type
diff --git a/sources/shiboken2/libshiboken/sbkstring.h b/sources/shiboken2/libshiboken/sbkstring.h
index c82ed5a22..84d7768c5 100644
--- a/sources/shiboken2/libshiboken/sbkstring.h
+++ b/sources/shiboken2/libshiboken/sbkstring.h
@@ -48,6 +48,7 @@ namespace Shiboken
 namespace String
 {
     LIBSHIBOKEN_API bool check(PyObject *obj);
+    LIBSHIBOKEN_API bool checkIterable(PyObject *obj);
     LIBSHIBOKEN_API bool checkType(PyTypeObject *obj);
     LIBSHIBOKEN_API bool checkChar(PyObject *obj);
     LIBSHIBOKEN_API bool isConvertible(PyObject *obj);
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
index efe4d26e2..2110ebe7a 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
@@ -197,7 +197,7 @@ class _Parameterized(object):
 class ResultVariable(_Parameterized):
     pass
 
-# Mark the primitive variables to become Sequence, Iterator or List
+# Mark the primitive variables to become Sequence, Iterable or List
 # (decided in the parser).
 class ArrayLikeVariable(_Parameterized):
     pass