rhi: Fix up vertex inputs with matrices

In order to prevent too much voodoo in backends like D3D11, the input
layout is expected to specify the slice index for vecX that are part of
an unrolled matrix.

Also deoptimize the instancing manual test to exercise a matrix too
instead of just vectors.

Change-Id: If2dcbcbc483645ce2420b2f87dda765b95da6e80
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

7 files changed, 51 insertions, 25 deletions

diff --git a/src/gui/rhi/qrhi.cpp b/src/gui/rhi/qrhi.cpp
index 06781686fa..5db0791322 100644
--- a/src/gui/rhi/qrhi.cpp
+++ b/src/gui/rhi/qrhi.cpp
@@ -1202,12 +1202,19 @@ QDebug operator<<(QDebug dbg, const QRhiVertexInputBinding &b)
 /*!
     Constructs a vertex input attribute description with the specified \a
     binding number, \a location, \a format, and \a offset.
+
+    \a matrixSlice should be -1 except when this attribute corresponds to a row
+    or column of a matrix (for example, a 4x4 matrix becomes 4 vec4s, consuming
+    4 consecutive vertex input locations), in which case it is the index of the
+    row or column. \c{location - matrixSlice} must always be equal to the \c
+    location for the first row or column of the unrolled matrix.
  */
-QRhiVertexInputAttribute::QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset)
+QRhiVertexInputAttribute::QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset, int matrixSlice)
     : m_binding(binding),
       m_location(location),
       m_format(format),
-      m_offset(offset)
+      m_offset(offset),
+      m_matrixSlice(matrixSlice)
 {
 }
 
diff --git a/src/gui/rhi/qrhi_p.h b/src/gui/rhi/qrhi_p.h
index 977b68cd8e..6a141d13e1 100644
--- a/src/gui/rhi/qrhi_p.h
+++ b/src/gui/rhi/qrhi_p.h
@@ -212,7 +212,7 @@ public:
     };
 
     QRhiVertexInputAttribute() = default;
-    QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset);
+    QRhiVertexInputAttribute(int binding, int location, Format format, quint32 offset, int matrixSlice = -1);
 
     int binding() const { return m_binding; }
     void setBinding(int b) { m_binding = b; }
@@ -226,11 +226,15 @@ public:
     quint32 offset() const { return m_offset; }
     void setOffset(quint32 ofs) { m_offset = ofs; }
 
+    int matrixSlice() const { return m_matrixSlice; }
+    void setMatrixSlice(int slice) { m_matrixSlice = slice; }
+
 private:
     int m_binding = 0;
     int m_location = 0;
     Format m_format = Float4;
     quint32 m_offset = 0;
+    int m_matrixSlice = -1;
 };
 
 Q_DECLARE_TYPEINFO(QRhiVertexInputAttribute, Q_MOVABLE_TYPE);
diff --git a/src/gui/rhi/qrhid3d11.cpp b/src/gui/rhi/qrhid3d11.cpp
index 48e626106c..ce954c9fe0 100644
--- a/src/gui/rhi/qrhid3d11.cpp
+++ b/src/gui/rhi/qrhid3d11.cpp
@@ -4019,15 +4019,29 @@ bool QD3D11GraphicsPipeline::create()
     d3dTopology = toD3DTopology(m_topology);
 
     if (!vsByteCode.isEmpty()) {
+        QByteArrayList matrixSliceSemantics;
         QVarLengthArray<D3D11_INPUT_ELEMENT_DESC, 4> inputDescs;
         for (auto it = m_vertexInputLayout.cbeginAttributes(), itEnd = m_vertexInputLayout.cendAttributes();
              it != itEnd; ++it)
         {
             D3D11_INPUT_ELEMENT_DESC desc;
             memset(&desc, 0, sizeof(desc));
-            // the output from SPIRV-Cross uses TEXCOORD<location> as the semantic
-            desc.SemanticName = "TEXCOORD";
-            desc.SemanticIndex = UINT(it->location());
+            // The output from SPIRV-Cross uses TEXCOORD<location> as the
+            // semantic, except for matrices that are unrolled into consecutive
+            // vec2/3/4s attributes and need TEXCOORD<location>_ as
+            // SemanticName and row/column index as SemanticIndex.
+            const int matrixSlice = it->matrixSlice();
+            if (matrixSlice < 0) {
+                desc.SemanticName = "TEXCOORD";
+                desc.SemanticIndex = UINT(it->location());
+            } else {
+                QByteArray sem;
+                sem.resize(16);
+                qsnprintf(sem.data(), sem.size(), "TEXCOORD%d_", it->location() - matrixSlice);
+                matrixSliceSemantics.append(sem);
+                desc.SemanticName = matrixSliceSemantics.last().constData();
+                desc.SemanticIndex = UINT(matrixSlice);
+            }
             desc.Format = toD3DAttributeFormat(it->format());
             desc.InputSlot = UINT(it->binding());
             desc.AlignedByteOffset = it->offset();
diff --git a/tests/manual/rhi/instancing/inst.frag.qsb b/tests/manual/rhi/instancing/inst.frag.qsb
index cd6b948227..c2a738734f 100644
--- a/tests/manual/rhi/instancing/inst.frag.qsb
+++ b/tests/manual/rhi/instancing/inst.frag.qsb
diff --git a/tests/manual/rhi/instancing/inst.vert b/tests/manual/rhi/instancing/inst.vert
index 3e1b117d58..c5a1e65e1b 100644
--- a/tests/manual/rhi/instancing/inst.vert
+++ b/tests/manual/rhi/instancing/inst.vert
@@ -2,9 +2,9 @@
 
 layout(location = 0) in vec4 position;
 
-// Instanced attributes to variate the translation and color of the cube
-layout(location = 1) in vec3 instTranslate;
-layout(location = 2) in vec3 instColor;
+// Instanced attributes to variate the transform and color of the cube
+layout(location = 1) in mat4 instMat;
+layout(location = 5) in vec3 instColor;
 
 layout(location = 0) out vec3 vColor;
 
@@ -17,9 +17,5 @@ layout(std140, binding = 0) uniform buf {
 void main()
 {
     vColor = instColor;
-    mat4 t = mat4(1, 0, 0, 0,
-                  0, 1, 0, 0,
-                  0, 0, 1, 0,
-                  instTranslate.x, instTranslate.y, instTranslate.z, 1);
-    gl_Position = ubuf.mvp * t * position;
+    gl_Position = ubuf.mvp * instMat * position;
 }
diff --git a/tests/manual/rhi/instancing/inst.vert.qsb b/tests/manual/rhi/instancing/inst.vert.qsb
index 4b891636e7..d9420639f8 100644
--- a/tests/manual/rhi/instancing/inst.vert.qsb
+++ b/tests/manual/rhi/instancing/inst.vert.qsb
diff --git a/tests/manual/rhi/instancing/instancing.cpp b/tests/manual/rhi/instancing/instancing.cpp
index 87a99b4581..5eb6293b6e 100644
--- a/tests/manual/rhi/instancing/instancing.cpp
+++ b/tests/manual/rhi/instancing/instancing.cpp
@@ -83,8 +83,8 @@ void Window::customInit()
 
     d.initialUpdates->uploadStaticBuffer(d.vbuf, cube);
 
-    // translation + color (vec3 + vec3), interleaved, for each instance
-    d.instBuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, INSTANCE_COUNT * 6 * sizeof(float));
+    // transform + color (mat4 + vec3), interleaved, for each instance
+    d.instBuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, INSTANCE_COUNT * 19 * sizeof(float));
     d.instBuf->create();
     d.releasePool << d.instBuf;
 
@@ -108,12 +108,15 @@ void Window::customInit()
     QRhiVertexInputLayout inputLayout;
     inputLayout.setBindings({
         { 3 * sizeof(float) },                                        // cube vertices
-        { 6 * sizeof(float), QRhiVertexInputBinding::PerInstance }    // per-instance translation and color
+        { 19 * sizeof(float), QRhiVertexInputBinding::PerInstance },  // per-instance transform and color
     });
     inputLayout.setAttributes({
-        { 0, 0, QRhiVertexInputAttribute::Float3, 0 },                // position
-        { 1, 1, QRhiVertexInputAttribute::Float3, 0 },                // instTranslate
-        { 1, 2, QRhiVertexInputAttribute::Float3, 3 * sizeof(float) } // instColor
+        { 0, 0, QRhiVertexInputAttribute::Float3, 0 },                     // position
+        { 1, 1, QRhiVertexInputAttribute::Float4, 0, 0 },                  // instMat
+        { 1, 2, QRhiVertexInputAttribute::Float4, 4 * sizeof(float), 1 },
+        { 1, 3, QRhiVertexInputAttribute::Float4, 8 * sizeof(float), 2 },
+        { 1, 4, QRhiVertexInputAttribute::Float4, 12 * sizeof(float), 3 },
+        { 1, 5, QRhiVertexInputAttribute::Float3, 16 * sizeof(float) },     // instColor
     });
     d.ps->setVertexInputLayout(inputLayout);
     d.ps->setShaderResourceBindings(d.srb);
@@ -121,14 +124,16 @@ void Window::customInit()
     d.ps->create();
 
     QByteArray instData;
-    instData.resize(INSTANCE_COUNT * 6 * sizeof(float));
+    instData.resize(INSTANCE_COUNT * 19 * sizeof(float));
     float *p = reinterpret_cast<float *>(instData.data());
     QRandomGenerator *rgen = QRandomGenerator::global();
     for (int i = 0; i < INSTANCE_COUNT; ++i) {
-        // translation
-        *p++ = rgen->bounded(8000) / 100.0f - 40.0f;
-        *p++ = rgen->bounded(8000) / 100.0f - 40.0f;
-        *p++ = 0.0f;
+        QMatrix4x4 m;
+        m.translate(rgen->bounded(8000) / 100.0f - 40.0f,
+                    rgen->bounded(8000) / 100.0f - 40.0f,
+                    0.0f);
+        memcpy(p, m.constData(), 16 * sizeof(float));
+        p += 16;
         // color
         *p++ = i / float(INSTANCE_COUNT);
         *p++ = 0.0f;