rhi: Fix up vertex inputs with matrices

In order to prevent too much voodoo in backends like D3D11, the input
layout is expected to specify the slice index for vecX that are part of
an unrolled matrix.

Also deoptimize the instancing manual test to exercise a matrix too
instead of just vectors.

Change-Id: If2dcbcbc483645ce2420b2f87dda765b95da6e80
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

3 files changed, 31 insertions, 6 deletions

diff --git a/src/gui/rhi/qrhi.cpp b/src/gui/rhi/qrhi.cpp
index 06781686fa..5db0791322 100644
--- a/src/gui/rhi/qrhi.cpp
+++ b/src/gui/rhi/qrhi.cpp
@@ -1202,12 +1202,19 @@ QDebug operator<<(QDebug dbg, const QRhiVertexInputBinding &b)
 /*!
     Constructs a vertex input attribute description with the specified \a
     binding number, \a location, \a format, and \a offset.
+
+    \a matrixSlice should be -1 except when this attribute corresponds to a row
+    or column of a matrix (for example, a 4x4 matrix becomes 4 vec4s, consuming
+    4 consecutive vertex input locations), in which case it is the index of the
+    row or column. \c{location - matrixSlice} must always be equal to the \c
+    location for the first row or column of the unrolled matrix.
  */
-QRhiVertexInputAttribute::QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset)
+QRhiVertexInputAttribute::QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset, int matrixSlice)
     : m_binding(binding),
       m_location(location),
       m_format(format),
-      m_offset(offset)
+      m_offset(offset),
+      m_matrixSlice(matrixSlice)
 {
 }
 
diff --git a/src/gui/rhi/qrhi_p.h b/src/gui/rhi/qrhi_p.h
index 977b68cd8e..6a141d13e1 100644
--- a/src/gui/rhi/qrhi_p.h
+++ b/src/gui/rhi/qrhi_p.h
@@ -212,7 +212,7 @@ public:
     };
 
     QRhiVertexInputAttribute() = default;
-    QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset);
+    QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset, int matrixSlice = -1);
 
     int binding() const { return m_binding; }
     void setBinding(int b) { m_binding = b; }
@@ -226,11 +226,15 @@ public:
     quint32 offset() const { return m_offset; }
     void setOffset(quint32 ofs) { m_offset = ofs; }
 
+    int matrixSlice() const { return m_matrixSlice; }
+    void setMatrixSlice(int slice) { m_matrixSlice = slice; }
+
 private:
     int m_binding = 0;
     int m_location = 0;
     Format m_format = Float4;
     quint32 m_offset = 0;
+    int m_matrixSlice = -1;
 };
 
 Q_DECLARE_TYPEINFO(QRhiVertexInputAttribute, Q_MOVABLE_TYPE);
diff --git a/src/gui/rhi/qrhid3d11.cpp b/src/gui/rhi/qrhid3d11.cpp
index 48e626106c..ce954c9fe0 100644
--- a/src/gui/rhi/qrhid3d11.cpp
+++ b/src/gui/rhi/qrhid3d11.cpp
@@ -4019,15 +4019,29 @@ bool QD3D11GraphicsPipeline::create()
     d3dTopology = toD3DTopology(m_topology);
 
     if (!vsByteCode.isEmpty()) {
+        QByteArrayList matrixSliceSemantics;
         QVarLengthArray<D3D11_INPUT_ELEMENT_DESC, 4> inputDescs;
         for (auto it = m_vertexInputLayout.cbeginAttributes(), itEnd = m_vertexInputLayout.cendAttributes();
              it != itEnd; ++it)
         {
             D3D11_INPUT_ELEMENT_DESC desc;
             memset(&desc, 0, sizeof(desc));
-            // the output from SPIRV-Cross uses TEXCOORD<location> as the semantic
-            desc.SemanticName = "TEXCOORD";
-            desc.SemanticIndex = UINT(it->location());
+            // The output from SPIRV-Cross uses TEXCOORD<location> as the
+            // semantic, except for matrices that are unrolled into consecutive
+            // vec2/3/4s attributes and need TEXCOORD<location>_ as
+            // SemanticName and row/column index as SemanticIndex.
+            const int matrixSlice = it->matrixSlice();
+            if (matrixSlice < 0) {
+                desc.SemanticName = "TEXCOORD";
+                desc.SemanticIndex = UINT(it->location());
+            } else {
+                QByteArray sem;
+                sem.resize(16);
+                qsnprintf(sem.data(), sem.size(), "TEXCOORD%d_", it->location() - matrixSlice);
+                matrixSliceSemantics.append(sem);
+                desc.SemanticName = matrixSliceSemantics.last().constData();
+                desc.SemanticIndex = UINT(matrixSlice);
+            }
             desc.Format = toD3DAttributeFormat(it->format());
             desc.InputSlot = UINT(it->binding());
             desc.AlignedByteOffset = it->offset();