rhi: Fix up vertex inputs with matrices

In order to prevent too much voodoo in backends like D3D11, the input
layout is expected to specify the slice index for vecX that are part of
an unrolled matrix.

Also deoptimize the instancing manual test to exercise a matrix too
instead of just vectors.

Change-Id: If2dcbcbc483645ce2420b2f87dda765b95da6e80
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

4 files changed, 20 insertions, 19 deletions

diff --git a/tests/manual/rhi/instancing/inst.frag.qsb b/tests/manual/rhi/instancing/inst.frag.qsb
index cd6b948227..c2a738734f 100644
--- a/tests/manual/rhi/instancing/inst.frag.qsb
+++ b/tests/manual/rhi/instancing/inst.frag.qsb
diff --git a/tests/manual/rhi/instancing/inst.vert b/tests/manual/rhi/instancing/inst.vert
index 3e1b117d58..c5a1e65e1b 100644
--- a/tests/manual/rhi/instancing/inst.vert
+++ b/tests/manual/rhi/instancing/inst.vert
@@ -2,9 +2,9 @@
 
 layout(location = 0) in vec4 position;
 
-// Instanced attributes to variate the translation and color of the cube
-layout(location = 1) in vec3 instTranslate;
-layout(location = 2) in vec3 instColor;
+// Instanced attributes to variate the transform and color of the cube
+layout(location = 1) in mat4 instMat;
+layout(location = 5) in vec3 instColor;
 
 layout(location = 0) out vec3 vColor;
 
@@ -17,9 +17,5 @@ layout(std140, binding = 0) uniform buf {
 void main()
 {
     vColor = instColor;
-    mat4 t = mat4(1, 0, 0, 0,
-                  0, 1, 0, 0,
-                  0, 0, 1, 0,
-                  instTranslate.x, instTranslate.y, instTranslate.z, 1);
-    gl_Position = ubuf.mvp * t * position;
+    gl_Position = ubuf.mvp * instMat * position;
 }
diff --git a/tests/manual/rhi/instancing/inst.vert.qsb b/tests/manual/rhi/instancing/inst.vert.qsb
index 4b891636e7..d9420639f8 100644
--- a/tests/manual/rhi/instancing/inst.vert.qsb
+++ b/tests/manual/rhi/instancing/inst.vert.qsb
diff --git a/tests/manual/rhi/instancing/instancing.cpp b/tests/manual/rhi/instancing/instancing.cpp
index 87a99b4581..5eb6293b6e 100644
--- a/tests/manual/rhi/instancing/instancing.cpp
+++ b/tests/manual/rhi/instancing/instancing.cpp
@@ -83,8 +83,8 @@ void Window::customInit()
 
     d.initialUpdates->uploadStaticBuffer(d.vbuf, cube);
 
-    // translation + color (vec3 + vec3), interleaved, for each instance
-    d.instBuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, INSTANCE_COUNT * 6 * sizeof(float));
+    // transform + color (mat4 + vec3), interleaved, for each instance
+    d.instBuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, INSTANCE_COUNT * 19 * sizeof(float));
     d.instBuf->create();
     d.releasePool << d.instBuf;
 
@@ -108,12 +108,15 @@ void Window::customInit()
     QRhiVertexInputLayout inputLayout;
     inputLayout.setBindings({
         { 3 * sizeof(float) },                                        // cube vertices
-        { 6 * sizeof(float), QRhiVertexInputBinding::PerInstance }    // per-instance translation and color
+        { 19 * sizeof(float), QRhiVertexInputBinding::PerInstance },  // per-instance transform and color
     });
     inputLayout.setAttributes({
-        { 0, 0, QRhiVertexInputAttribute::Float3, 0 },                // position
-        { 1, 1, QRhiVertexInputAttribute::Float3, 0 },                // instTranslate
-        { 1, 2, QRhiVertexInputAttribute::Float3, 3 * sizeof(float) } // instColor
+        { 0, 0, QRhiVertexInputAttribute::Float3, 0 },                     // position
+        { 1, 1, QRhiVertexInputAttribute::Float4, 0, 0 },                  // instMat
+        { 1, 2, QRhiVertexInputAttribute::Float4, 4 * sizeof(float), 1 },
+        { 1, 3, QRhiVertexInputAttribute::Float4, 8 * sizeof(float), 2 },
+        { 1, 4, QRhiVertexInputAttribute::Float4, 12 * sizeof(float), 3 },
+        { 1, 5, QRhiVertexInputAttribute::Float3, 16 * sizeof(float) },     // instColor
     });
     d.ps->setVertexInputLayout(inputLayout);
     d.ps->setShaderResourceBindings(d.srb);
@@ -121,14 +124,16 @@ void Window::customInit()
     d.ps->create();
 
     QByteArray instData;
-    instData.resize(INSTANCE_COUNT * 6 * sizeof(float));
+    instData.resize(INSTANCE_COUNT * 19 * sizeof(float));
     float *p = reinterpret_cast<float *>(instData.data());
     QRandomGenerator *rgen = QRandomGenerator::global();
     for (int i = 0; i < INSTANCE_COUNT; ++i) {
-        // translation
-        *p++ = rgen->bounded(8000) / 100.0f - 40.0f;
-        *p++ = rgen->bounded(8000) / 100.0f - 40.0f;
-        *p++ = 0.0f;
+        QMatrix4x4 m;
+        m.translate(rgen->bounded(8000) / 100.0f - 40.0f,
+                    rgen->bounded(8000) / 100.0f - 40.0f,
+                    0.0f);
+        memcpy(p, m.constData(), 16 * sizeof(float));
+        p += 16;
         // color
         *p++ = i / float(INSTANCE_COUNT);
         *p++ = 0.0f;