Support Pointer Primitive Types by Arrays or Result Tuples

-- This change is part of the improved numpy support --

Most primitive types are handled in XML, but this was not reflected
by the signatures, error messages, doc strings and hinting stubs.

In order to enhance the information shown to be more correct,
the C++ parser part was rewritten for Python. It is written
closely to Python syntax, but keeps the existing information about
primitive types intact.

AbstractMetaType::NativePointerAsArrayPattern is now used to
mark a variable as an array. Heuristics are no longer used.

If a pointer variable is not marked as an array, the Python parser
generates a return value. If more than one value would be returned,
a result-tuple is generated.

Because we now have a deterministic categorization of types, the
"const" attribute is no more needed and the entries in mapping.py
are reduced.
A few missing <array/> markers were added.

The tool also now handles typing.List[] differently in arguments and
return types. While return types stay lists, they are for now changed
to typing.Sequence[] in argument lists.

A test was included.

These messages belong to the previous "deprecated functions" patch:

Further, QMatrixMxN.constData was removed from the typesystem
and replaced by a surrogate function that calls QMatrixMxN.data,
but also generates a warning.

The long forgotten generate_pyi.py was now published in the same
course.

Task-number: PYSIDE-795
Task-number: PYSIDE-951
Change-Id: Ia59fe4986919525a70ea7cc453c64cdf46e7fba0
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

2 files changed, 208 insertions, 73 deletions

diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
index 36b137104..5d6a24016 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
@@ -65,7 +65,6 @@ Point = typing.Tuple[float, float]
 Variant = typing.Any
 ModelIndexList = typing.List[int]
 QImageCleanupFunction = typing.Callable
-StringList = typing.List[str]
 
 _S = TypeVar("_S")
 
@@ -195,6 +194,27 @@ class Default(_NotCalled):
 class Instance(_NotCalled):
     pass
 
+# Parameterized primitive variables
+class _Parameterized(object):
+    def __init__(self, type):
+        self.type = type
+        self.__name__ = self.__class__.__name__
+
+    def __repr__(self):
+        return "{}({})".format(
+            type(self).__name__, self.type.__name__)
+
+# Mark the primitive variables to be moved into the result.
+class ResultVariable(_Parameterized):
+    pass
+
+# Mark the primitive variables to become Sequence, Iterator or List
+# (decided in the parser).
+class ArrayLikeVariable(_Parameterized):
+    pass
+
+StringList = ArrayLikeVariable(str)
+
 
 class Reloader(object):
     """
@@ -262,7 +282,7 @@ type_map.update({
     "double": float,
     "float": float,
     "int": int,
-    "List": typing.List,
+    "List": ArrayLikeVariable,
     "long": int,
     "PyCallable": typing.Callable,
     "PyObject": object,
@@ -275,7 +295,7 @@ type_map.update({
     "qint64": int,
     "qint8": int,
     "qintptr": int,
-    "QList": typing.List,
+    "QList": ArrayLikeVariable,
     "qlonglong": int,
     "QMap": typing.Dict,
     "QPair": typing.Tuple,
@@ -297,17 +317,27 @@ type_map.update({
     "short": int,
     "signed char": Char,
     "signed long": int,
+    "std.list": typing.List,
+    "std.map": typing.Dict,
+    "std.pair": typing.Tuple,
+    "std.vector": typing.List,
     "str": str,
     "true": True,
+    "Tuple": typing.Tuple,
+    "uchar": Char,
+    "uchar*": str,
+    "uint": int,
+    "ulong": int,
     "ULONG_MAX": ulong_max,
-    "unsigned char": Char,
-    "unsigned int": int, # should we define an unsigned type?
+    "unsigned char": Char, # 5.9
+    "unsigned char*": str,
+    "unsigned int": int,
     "unsigned long int": int, # 5.6, RHEL 6.6
     "unsigned long long": int,
     "unsigned long": int,
     "unsigned short int": int, # 5.6, RHEL 6.6
     "unsigned short": int,
-    "UnsignedShortType": int, # 5.9
+    "ushort": int,
     "void": int, # be more specific?
     "WId": WId,
     "zero(bytes)": b"",
@@ -316,6 +346,50 @@ type_map.update({
     "zero(int)": 0,
     "zero(object)": None,
     "zero(str)": "",
+    "zero(typing.Any)": None,
+    })
+
+type_map.update({
+    # Handling variables declared as array:
+    "array double*"         : ArrayLikeVariable(float),
+    "array float*"          : ArrayLikeVariable(float),
+    "array GLint*"          : ArrayLikeVariable(int),
+    "array GLuint*"         : ArrayLikeVariable(int),
+    "array int*"            : ArrayLikeVariable(int),
+    "array long long*"      : ArrayLikeVariable(int),
+    "array short*"          : ArrayLikeVariable(int),
+    "array signed char*"    : bytes,
+    "array unsigned char*"  : bytes,
+    "array unsigned int*"   : ArrayLikeVariable(int),
+    "array unsigned short*" : ArrayLikeVariable(int),
+    })
+
+type_map.update({
+    # Special cases:
+    "char*"         : bytes,
+    "QChar*"        : bytes,
+    "quint32*"      : int,        # only for QRandomGenerator
+    "quint8*"       : bytearray,  # only for QCborStreamReader and QCborValue
+    "uchar*"        : bytes,
+    "unsigned char*": bytes,
+    })
+
+type_map.update({
+    # Handling variables that are returned, eventually as Tuples:
+    "bool*"         : ResultVariable(bool),
+    "float*"        : ResultVariable(float),
+    "int*"          : ResultVariable(int),
+    "long long*"    : ResultVariable(int),
+    "long*"         : ResultVariable(int),
+    "PStr*"         : ResultVariable(str),  # module sample
+    "qint32*"       : ResultVariable(int),
+    "qint64*"       : ResultVariable(int),
+    "qreal*"        : ResultVariable(float),
+    "QString*"      : ResultVariable(str),
+    "quint16*"      : ResultVariable(int),
+    "uint*"         : ResultVariable(int),
+    "unsigned int*" : ResultVariable(int),
+    "QStringList*"  : ResultVariable(StringList),
     })
 
 
@@ -340,6 +414,7 @@ def init_sample():
     import datetime
     type_map.update({
         "char": Char,
+        "char**": typing.List[str],
         "Complex": complex,
         "double": float,
         "Foo.HANDLE": int,
@@ -355,7 +430,8 @@ def init_sample():
         "sample.int": int,
         "sample.ObjectType": object,
         "sample.OddBool": bool,
-        "sample.Photon.TemplateBase": Missing("sample.Photon.TemplateBase"),
+        "sample.Photon.TemplateBase[Photon.DuplicatorType]": sample.Photon.ValueDuplicator,
+        "sample.Photon.TemplateBase[Photon.IdentityType]": sample.Photon.ValueIdentity,
         "sample.Point": Point,
         "sample.PStr": str,
         "sample.unsigned char": Char,
@@ -390,6 +466,7 @@ def init_smart():
     })
     return locals()
 
+
 # The PySide Part
 def init_PySide2_QtCore():
     from PySide2.QtCore import Qt, QUrl, QDir
@@ -411,50 +488,18 @@ def init_PySide2_QtCore():
         "list of QAbstractAnimation": typing.List[PySide2.QtCore.QAbstractAnimation],
         "list of QAbstractState": typing.List[PySide2.QtCore.QAbstractState],
         "long long": int,
-        "long": int,
         "NULL": None, # 5.6, MSVC
         "nullptr": None, # 5.9
         "PyByteArray": bytearray,
         "PyBytes": bytes,
-        "PyCallable": typing.Callable,
-        "PyObject": object,
-        "PySequence": typing.Iterable,  # important for numpy
-        "PySide2.QtCore.bool": bool,
-        "PySide2.QtCore.char": StringList, # A 'char **' is a list of strings.
-        "PySide2.QtCore.double": float,
-        "PySide2.QtCore.float": float,
-        "PySide2.QtCore.int": int,
-        "PySide2.QtCore.int32_t": int, # 5.9
-        "PySide2.QtCore.int64_t": int, # 5.9
-        "PySide2.QtCore.long long": int, # 5.9, MSVC 15
-        "PySide2.QtCore.long": int,
-        "PySide2.QtCore.QCborStreamReader.StringResult": typing.AnyStr,
-        "PySide2.QtCore.QChar": Char,
-        "PySide2.QtCore.qint16": int,
-        "PySide2.QtCore.qint32": int,
-        "PySide2.QtCore.qint64": int,
-        "PySide2.QtCore.qint8": int,
-        "PySide2.QtCore.qreal": float,
-        "PySide2.QtCore.QString": str,
-        "PySide2.QtCore.QStringList": StringList,
-        "PySide2.QtCore.quint16": int,
-        "PySide2.QtCore.quint32": int,
-        "PySide2.QtCore.quint64": int,
-        "PySide2.QtCore.quint8": int,
+        "PySide2.QtCore.QCborStreamReader.StringResult[PySide2.QtCore.QByteArray]":
+            PySide2.QtCore.QCborStringResultByteArray,
+        "PySide2.QtCore.QCborStreamReader.StringResult[QString]":
+            PySide2.QtCore.QCborStringResultString,
         "PySide2.QtCore.QUrl.ComponentFormattingOptions":
             PySide2.QtCore.QUrl.ComponentFormattingOption, # mismatch option/enum, why???
-        "PySide2.QtCore.QVariant": Variant,
-        "PySide2.QtCore.short": int,
-        "PySide2.QtCore.signed char": Char,
-        "PySide2.QtCore.uchar": Char,
-        "PySide2.QtCore.uint32_t": int, # 5.9
-        "PySide2.QtCore.unsigned char": Char, # 5.9
-        "PySide2.QtCore.unsigned int": int, # 5.9 Ubuntu
-        "PySide2.QtCore.unsigned short": int,
-        "PyTypeObject": type,
         "PyUnicode": typing.Text,
         "Q_NULLPTR": None,
-        "QChar": Char,
         "QDir.Filters(AllEntries | NoDotAndDotDot)": Instance(
             "QDir.Filters(QDir.AllEntries | QDir.NoDotAndDotDot)"),
         "QDir.SortFlags(Name | IgnoreCase)": Instance(
@@ -465,24 +510,21 @@ def init_PySide2_QtCore():
         "QGenericArgument(NULL)": ellipsis, # 5.6, MSVC
         "QGenericArgument(nullptr)": ellipsis, # 5.10
         "QGenericArgument(Q_NULLPTR)": ellipsis,
-        "QHash": typing.Dict,
         "QJsonObject": typing.Dict[str, PySide2.QtCore.QJsonValue],
         "QModelIndex()": Invalid("PySide2.QtCore.QModelIndex"), # repr is btw. very wrong, fix it?!
         "QModelIndexList": ModelIndexList,
-        "qptrdiff": int,
-        "QString": str,
+        "QModelIndexList": ModelIndexList,
         "QString()": "",
-        "QStringList": StringList,
         "QStringList()": [],
         "QStringRef": str,
-        "Qt.HANDLE": int, # be more explicit with some consts?
-        "quintptr": int,
+        "QStringRef": str,
+        "Qt.HANDLE": int, # be more explicit with some constants?
         "QUrl.FormattingOptions(PrettyDecoded)": Instance(
             "QUrl.FormattingOptions(QUrl.PrettyDecoded)"),
-        "QVariant": Variant,
         "QVariant()": Invalid(Variant),
         "QVariant.Type": type, # not so sure here...
         "QVariantMap": typing.Dict[str, Variant],
+        "QVariantMap": typing.Dict[str, Variant],
     })
     try:
         type_map.update({
@@ -502,16 +544,12 @@ def init_PySide2_QtGui():
         "GL_COLOR_BUFFER_BIT": GL_COLOR_BUFFER_BIT,
         "GL_NEAREST": GL_NEAREST,
         "int32_t": int,
-        "PySide2.QtCore.uint8_t": int, # macOS 5.9
-        "PySide2.QtGui.QGenericMatrix": Missing("PySide2.QtGui.QGenericMatrix"),
-        "PySide2.QtGui.QPlatformSurface": int, # a handle
-        "QList< QTouchEvent.TouchPoint >()": [], # XXX improve?
         "QPixmap()": Default("PySide2.QtGui.QPixmap"), # can't create without qApp
+        "QPlatformSurface*": int, # a handle
         "QVector< QTextLayout.FormatRange >()": [], # do we need more structure?
         "uint32_t": int,
         "uint8_t": int,
         "USHRT_MAX": ushort_max,
-        "WId": WId,
     })
     return locals()
 
@@ -522,7 +560,6 @@ def init_PySide2_QtWidgets():
     type_map.update({
         "QMessageBox.StandardButtons(Yes | No)": Instance(
             "QMessageBox.StandardButtons(QMessageBox.Yes | QMessageBox.No)"),
-        "QVector< int >()": [],
         "QWidget.RenderFlags(DrawWindowBackground | DrawChildren)": Instance(
             "QWidget.RenderFlags(QWidget.DrawWindowBackground | QWidget.DrawChildren)"),
         "SH_Default": QStyleHintReturn.SH_Default,
@@ -545,9 +582,12 @@ def init_PySide2_QtSql():
 
 
 def init_PySide2_QtNetwork():
+    best_structure = typing.OrderedDict if getattr(typing, "OrderedDict", None) else typing.Dict
     type_map.update({
-        "QMultiMap": MultiMap,
+        "QMultiMap[PySide2.QtNetwork.QSsl.AlternativeNameEntryType, QString]":
+            best_structure[PySide2.QtNetwork.QSsl.AlternativeNameEntryType, typing.List[str]],
     })
+    del best_structure
     return locals()
 
 
@@ -566,6 +606,7 @@ def init_PySide2_QtMultimedia():
     check_module(PySide2.QtMultimediaWidgets)
     type_map.update({
         "QGraphicsVideoItem": PySide2.QtMultimediaWidgets.QGraphicsVideoItem,
+        "qint64": int,
         "QVideoWidget": PySide2.QtMultimediaWidgets.QVideoWidget,
     })
     return locals()
@@ -578,26 +619,23 @@ def init_PySide2_QtOpenGL():
         "GLfloat": float, # 5.6, MSVC 15
         "GLint": int,
         "GLuint": int,
-        "PySide2.QtOpenGL.GLint": int,
-        "PySide2.QtOpenGL.GLuint": int,
     })
     return locals()
 
 
 def init_PySide2_QtQml():
     type_map.update({
-        "PySide2.QtQml.bool volatile": bool,
         "QJSValueList()": [],
-        "QVariantHash()": typing.Dict[str, Variant],  # XXX sorted?
+        "QVariantHash()": typing.Dict[str, Variant],    # from 5.9
     })
     return locals()
 
 
 def init_PySide2_QtQuick():
     type_map.update({
-        "PySide2.QtCore.uint": int,
-        "PySide2.QtQuick.QSharedPointer": int,
-        "T": int,
+        "PySide2.QtQuick.QSharedPointer[PySide2.QtQuick.QQuickItemGrabResult]":
+            PySide2.QtQuick.QQuickItemGrabResult,
+        "UnsignedShortType": int,
     })
     return locals()
 
@@ -611,6 +649,7 @@ def init_PySide2_QtScript():
 
 def init_PySide2_QtTest():
     type_map.update({
+        "PySide2.QtTest.QTest.PySideQTouchEventSequence": PySide2.QtTest.QTest.QTouchEventSequence,
         "PySide2.QtTest.QTouchEventSequence": PySide2.QtTest.QTest.QTouchEventSequence,
     })
     return locals()
@@ -631,6 +670,10 @@ def init_PySide2_QtDataVisualization():
     QtDataVisualization.QSurfaceDataArray = typing.List[QtDataVisualization.QSurfaceDataRow]
     type_map.update({
         "100.0f": 100.0,
+        "QtDataVisualization.QBarDataArray": QtDataVisualization.QBarDataArray,
+        "QtDataVisualization.QBarDataArray*": QtDataVisualization.QBarDataArray,
+        "QtDataVisualization.QSurfaceDataArray": QtDataVisualization.QSurfaceDataArray,
+        "QtDataVisualization.QSurfaceDataArray*": QtDataVisualization.QSurfaceDataArray,
     })
     return locals()
 
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
index 6109bceee..204f37384 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
@@ -46,7 +46,7 @@ import types
 import keyword
 import functools
 from shibokensupport.signature.mapping import (type_map, update_mapping,
-    namespace, typing, _NotCalled)
+    namespace, typing, _NotCalled, ResultVariable, ArrayLikeVariable)
 from shibokensupport.signature.lib.tool import (SimpleNamespace,
     build_brace_pattern)
 
@@ -169,7 +169,9 @@ def _resolve_value(thing, valtype, line):
     if thing in ("0", "None") and valtype:
         if valtype.startswith("PySide2."):
             return None
-        name = type_map[valtype].__name__
+        map = type_map[valtype]
+        # typing.Any: '_SpecialForm' object has no attribute '__name__'
+        name = map.__name__ if hasattr(map, "__name__") else str(map)
         thing = "zero({})".format(name)
     if thing in type_map:
         return type_map[thing]
@@ -216,8 +218,6 @@ def to_string(thing):
 
 matrix_pattern = "PySide2.QtGui.QGenericMatrix"
 
-# The matrix patch is borrowed from the future (extracted).
-# It will work when the parser recognizes matrices.
 def handle_matrix(arg):
     n, m, typstr = tuple(map(lambda x:x.strip(), arg.split(",")))
     assert typstr == "float"
@@ -235,7 +235,7 @@ def lno(level):
 """
 
 
-def _resolve_type(thing, line, level):
+def _resolve_type(thing, line, level, var_handler):
     # Capture total replacements, first. Happens in
     # "PySide2.QtCore.QCborStreamReader.StringResult[PySide2.QtCore.QByteArray]"
     if thing in type_map:
@@ -250,13 +250,13 @@ def _resolve_type(thing, line, level):
         # Special case: Handle the generic matrices.
         if contr == matrix_pattern:
             return handle_matrix(thing)
-        contr = _resolve_type(contr, line, level+1)
+        contr = var_handler(_resolve_type(contr, line, level+1, var_handler))
         if isinstance(contr, _NotCalled):
             raise SystemError("Container types must exist:", repr(contr))
         contr = to_string(contr)
         pieces = []
         for part in _parse_arglist(thing):
-            part = _resolve_type(part, line, level+1)
+            part = var_handler(_resolve_type(part, line, level+1, var_handler))
             if isinstance(part, _NotCalled):
                 # fix the tag (i.e. "Missing") by repr
                 part = repr(part)
@@ -267,6 +267,46 @@ def _resolve_type(thing, line, level):
     return _resolve_value(thing, None, line)
 
 
+def _handle_generic(obj, repl):
+    """
+    Assign repl if obj is an ArrayLikeVariable
+
+    This is a neat trick. Example:
+
+        obj                     repl        result
+        ----------------------  --------    ---------
+        ArrayLikeVariable       List        List
+        ArrayLikeVariable(str)  List        List[str]
+        ArrayLikeVariable       Sequence    Sequence
+        ArrayLikeVariable(str)  Sequence    Sequence[str]
+    """
+    if isinstance(obj, ArrayLikeVariable):
+        return repl[obj.type]
+    if isinstance(obj, type) and issubclass(obj, ArrayLikeVariable):
+        # was "if obj is ArrayLikeVariable"
+        return repl
+    return obj
+
+
+def handle_argvar(obj):
+    """
+    Decide how array-like variables are resolved in arguments
+
+    Currently, the best approximation is types.Sequence.
+    We want to change that to types.Iterable in the near future.
+    """
+    return _handle_generic(obj, typing.Sequence)
+
+
+def handle_retvar(obj):
+    """
+    Decide how array-like variables are resolved in results
+
+    This will probably stay typing.List forever.
+    """
+    return _handle_generic(obj, typing.List)
+
+
 def calculate_props(line):
     parsed = SimpleNamespace(**_parse_line(line.strip()))
     arglist = parsed.arglist
@@ -280,14 +320,14 @@ def calculate_props(line):
             ann = 'NULL' # maps to None
             tup = name, ann
             arglist[idx] = tup
-        annotations[name] = _resolve_type(ann, line, 0)
+        annotations[name] = _resolve_type(ann, line, 0, handle_argvar)
         if len(tup) == 3:
             default = _resolve_value(tup[2], ann, line)
             _defaults.append(default)
     defaults = tuple(_defaults)
     returntype = parsed.returntype
     if returntype is not None:
-        annotations["return"] = _resolve_type(returntype, line, 0)
+        annotations["return"] = _resolve_type(returntype, line, 0, handle_retvar)
     props = SimpleNamespace()
     props.defaults = defaults
     props.kwdefaults = {}
@@ -298,9 +338,61 @@ def calculate_props(line):
     shortname = funcname[funcname.rindex(".")+1:]
     props.name = shortname
     props.multi = parsed.multi
+    fix_variables(props, line)
     return vars(props)
 
 
+def fix_variables(props, line):
+    annos = props.annotations
+    if not any(isinstance(ann, (ResultVariable, ArrayLikeVariable))
+               for ann in annos.values()):
+        return
+    retvar = annos.get("return", None)
+    if retvar and isinstance(retvar, (ResultVariable, ArrayLikeVariable)):
+        # Special case: a ResultVariable which is the result will always be an array!
+        annos["return"] = retvar = typing.List[retvar.type]
+    fullname = props.fullname
+    varnames = list(props.varnames)
+    defaults = list(props.defaults)
+    diff = len(varnames) - len(defaults)
+
+    safe_annos = annos.copy()
+    retvars = [retvar] if retvar else []
+    deletions = []
+    for idx, name in enumerate(varnames):
+        ann = safe_annos[name]
+        if isinstance(ann, ArrayLikeVariable):
+            ann = typing.Sequence[ann.type]
+            annos[name] = ann
+        if not isinstance(ann, ResultVariable):
+            continue
+        # We move the variable to the end and remove it.
+        retvars.append(ann.type)
+        deletions.append(idx)
+        del annos[name]
+    for idx in reversed(deletions):
+        # varnames:  0 1 2 3 4 5 6 7
+        # defaults:        0 1 2 3 4
+        # diff: 3
+        del varnames[idx]
+        if idx >= diff:
+            del defaults[idx - diff]
+        else:
+            diff -= 1
+    if retvars:
+        rvs = []
+        retvars = list(handle_retvar(rv) if isinstance(rv, ArrayLikeVariable) else rv
+                       for rv in retvars)
+        if len(retvars) == 1:
+            returntype = retvars[0]
+        else:
+            typestr = "typing.Tuple[{}]".format(", ".join(map(to_string, retvars)))
+            returntype = eval(typestr, namespace)
+        props.annotations["return"] = returntype
+    props.varnames = tuple(varnames)
+    props.defaults = tuple(defaults)
+
+
 def fixup_multilines(lines):
     """
     Multilines can collapse when certain distinctions between C++ types