Support Pointer Primitive Types by Arrays or Result Tuples

-- This change is part of the improved numpy support --

Most primitive types are handled in XML, but this was not reflected
by the signatures, error messages, doc strings and hinting stubs.

In order to enhance the information shown to be more correct,
the C++ parser part was rewritten for Python. It is written
closely to Python syntax, but keeps the existing information about
primitive types intact.

AbstractMetaType::NativePointerAsArrayPattern is now used to
mark a variable as an array. Heuristics are no longer used.

If a pointer variable is not marked as an array, the Python parser
generates a return value. If more than one value would be returned,
a result-tuple is generated.

Because we now have a deterministic categorization of types, the
"const" attribute is no more needed and the entries in mapping.py
are reduced.
A few missing <array/> markers were added.

The tool also now handles typing.List[] differently in arguments and
return types. While return types stay lists, they are for now changed
to typing.Sequence[] in argument lists.

A test was included.

These messages belong to the previous "deprecated functions" patch:

Further, QMatrixMxN.constData was removed from the typesystem
and replaced by a surrogate function that calls QMatrixMxN.data,
but also generates a warning.

The long forgotten generate_pyi.py was now published in the same
course.

Task-number: PYSIDE-795
Task-number: PYSIDE-951
Change-Id: Ia59fe4986919525a70ea7cc453c64cdf46e7fba0
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

1 files changed, 101 insertions, 9 deletions

diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
index 6109bceee..204f37384 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/parser.py
@@ -46,7 +46,7 @@ import types
 import keyword
 import functools
 from shibokensupport.signature.mapping import (type_map, update_mapping,
-    namespace, typing, _NotCalled)
+    namespace, typing, _NotCalled, ResultVariable, ArrayLikeVariable)
 from shibokensupport.signature.lib.tool import (SimpleNamespace,
     build_brace_pattern)
 
@@ -169,7 +169,9 @@ def _resolve_value(thing, valtype, line):
     if thing in ("0", "None") and valtype:
         if valtype.startswith("PySide2."):
             return None
-        name = type_map[valtype].__name__
+        map = type_map[valtype]
+        # typing.Any: '_SpecialForm' object has no attribute '__name__'
+        name = map.__name__ if hasattr(map, "__name__") else str(map)
         thing = "zero({})".format(name)
     if thing in type_map:
         return type_map[thing]
@@ -216,8 +218,6 @@ def to_string(thing):
 
 matrix_pattern = "PySide2.QtGui.QGenericMatrix"
 
-# The matrix patch is borrowed from the future (extracted).
-# It will work when the parser recognizes matrices.
 def handle_matrix(arg):
     n, m, typstr = tuple(map(lambda x:x.strip(), arg.split(",")))
     assert typstr == "float"
@@ -235,7 +235,7 @@ def lno(level):
 """
 
 
-def _resolve_type(thing, line, level):
+def _resolve_type(thing, line, level, var_handler):
     # Capture total replacements, first. Happens in
     # "PySide2.QtCore.QCborStreamReader.StringResult[PySide2.QtCore.QByteArray]"
     if thing in type_map:
@@ -250,13 +250,13 @@ def _resolve_type(thing, line, level):
         # Special case: Handle the generic matrices.
         if contr == matrix_pattern:
             return handle_matrix(thing)
-        contr = _resolve_type(contr, line, level+1)
+        contr = var_handler(_resolve_type(contr, line, level+1, var_handler))
         if isinstance(contr, _NotCalled):
             raise SystemError("Container types must exist:", repr(contr))
         contr = to_string(contr)
         pieces = []
         for part in _parse_arglist(thing):
-            part = _resolve_type(part, line, level+1)
+            part = var_handler(_resolve_type(part, line, level+1, var_handler))
             if isinstance(part, _NotCalled):
                 # fix the tag (i.e. "Missing") by repr
                 part = repr(part)
@@ -267,6 +267,46 @@ def _resolve_type(thing, line, level):
     return _resolve_value(thing, None, line)
 
 
+def _handle_generic(obj, repl):
+    """
+    Assign repl if obj is an ArrayLikeVariable
+
+    This is a neat trick. Example:
+
+        obj                     repl        result
+        ----------------------  --------    ---------
+        ArrayLikeVariable       List        List
+        ArrayLikeVariable(str)  List        List[str]
+        ArrayLikeVariable       Sequence    Sequence
+        ArrayLikeVariable(str)  Sequence    Sequence[str]
+    """
+    if isinstance(obj, ArrayLikeVariable):
+        return repl[obj.type]
+    if isinstance(obj, type) and issubclass(obj, ArrayLikeVariable):
+        # was "if obj is ArrayLikeVariable"
+        return repl
+    return obj
+
+
+def handle_argvar(obj):
+    """
+    Decide how array-like variables are resolved in arguments
+
+    Currently, the best approximation is types.Sequence.
+    We want to change that to types.Iterable in the near future.
+    """
+    return _handle_generic(obj, typing.Sequence)
+
+
+def handle_retvar(obj):
+    """
+    Decide how array-like variables are resolved in results
+
+    This will probably stay typing.List forever.
+    """
+    return _handle_generic(obj, typing.List)
+
+
 def calculate_props(line):
     parsed = SimpleNamespace(**_parse_line(line.strip()))
     arglist = parsed.arglist
@@ -280,14 +320,14 @@ def calculate_props(line):
             ann = 'NULL' # maps to None
             tup = name, ann
             arglist[idx] = tup
-        annotations[name] = _resolve_type(ann, line, 0)
+        annotations[name] = _resolve_type(ann, line, 0, handle_argvar)
         if len(tup) == 3:
             default = _resolve_value(tup[2], ann, line)
             _defaults.append(default)
     defaults = tuple(_defaults)
     returntype = parsed.returntype
     if returntype is not None:
-        annotations["return"] = _resolve_type(returntype, line, 0)
+        annotations["return"] = _resolve_type(returntype, line, 0, handle_retvar)
     props = SimpleNamespace()
     props.defaults = defaults
     props.kwdefaults = {}
@@ -298,9 +338,61 @@ def calculate_props(line):
     shortname = funcname[funcname.rindex(".")+1:]
     props.name = shortname
     props.multi = parsed.multi
+    fix_variables(props, line)
     return vars(props)
 
 
+def fix_variables(props, line):
+    annos = props.annotations
+    if not any(isinstance(ann, (ResultVariable, ArrayLikeVariable))
+               for ann in annos.values()):
+        return
+    retvar = annos.get("return", None)
+    if retvar and isinstance(retvar, (ResultVariable, ArrayLikeVariable)):
+        # Special case: a ResultVariable which is the result will always be an array!
+        annos["return"] = retvar = typing.List[retvar.type]
+    fullname = props.fullname
+    varnames = list(props.varnames)
+    defaults = list(props.defaults)
+    diff = len(varnames) - len(defaults)
+
+    safe_annos = annos.copy()
+    retvars = [retvar] if retvar else []
+    deletions = []
+    for idx, name in enumerate(varnames):
+        ann = safe_annos[name]
+        if isinstance(ann, ArrayLikeVariable):
+            ann = typing.Sequence[ann.type]
+            annos[name] = ann
+        if not isinstance(ann, ResultVariable):
+            continue
+        # We move the variable to the end and remove it.
+        retvars.append(ann.type)
+        deletions.append(idx)
+        del annos[name]
+    for idx in reversed(deletions):
+        # varnames:  0 1 2 3 4 5 6 7
+        # defaults:        0 1 2 3 4
+        # diff: 3
+        del varnames[idx]
+        if idx >= diff:
+            del defaults[idx - diff]
+        else:
+            diff -= 1
+    if retvars:
+        rvs = []
+        retvars = list(handle_retvar(rv) if isinstance(rv, ArrayLikeVariable) else rv
+                       for rv in retvars)
+        if len(retvars) == 1:
+            returntype = retvars[0]
+        else:
+            typestr = "typing.Tuple[{}]".format(", ".join(map(to_string, retvars)))
+            returntype = eval(typestr, namespace)
+        props.annotations["return"] = returntype
+    props.varnames = tuple(varnames)
+    props.defaults = tuple(defaults)
+
+
 def fixup_multilines(lines):
     """
     Multilines can collapse when certain distinctions between C++ types