rhi: d3d11: Enable tessellation and geometry with some caveats

The caveat being having to manually create HLSL versions of the hull,
domain, and geometry shaders in parallel with the Vulkan GLSL ones,
while keeping the interfaces intact (stage inputs and outputs, cbuffer
layouts, binding points/registers). This is not always trivial but
typically doable in not very complicated case after inspecting the
SPIRV-Cross-generated vertex/fragment code in the .qsb files. Once
written, the HLSL files can be injected into a .qsb file with qsb -r.
or the corresponding CMake syntax. Conceptually this is no different
from how samplerExternalOES support is implemented for Multimedia.
(there the problem is that the shaders cannot be compiled to SPIR-V
to begin with, here it is that we cannot translate from SPIR-V, but
in the end the workaround for both problems is effectively the same)

The manual tests demonstrate this, both the tessellation and geometry
apps work now with D3D out of the box.

On the bright side, the implementation here in the the D3D backend of
QRhi does not need to know about how the shaders got there in the
QShader. So none of the implementation is dependent on this manual
process. If some day qsb would start translating to these kind of
shaders as well, it would all still work as-is.

Change-Id: I32d9ab94e00174e4bd5b59ac814dfedef9f93ad1
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

12 files changed, 111 insertions, 4 deletions

diff --git a/tests/manual/rhi/geometryshader/buildshaders.bat b/tests/manual/rhi/geometryshader/buildshaders.bat
index e15ca63aea..6da26a6a2a 100755
--- a/tests/manual/rhi/geometryshader/buildshaders.bat
+++ b/tests/manual/rhi/geometryshader/buildshaders.bat
@@ -1,3 +1,4 @@
-qsb --glsl 320es,410 test.vert -o test.vert.qsb
+qsb --glsl 320es,410 --hlsl 50 test.vert -o test.vert.qsb
 qsb --glsl 320es,410 test.geom -o test.geom.qsb
-qsb --glsl 320es,410 test.frag -o test.frag.qsb
+qsb -r hlsl,50,test_geom.hlsl test.geom.qsb
+qsb --glsl 320es,410 --hlsl 50 test.frag -o test.frag.qsb
diff --git a/tests/manual/rhi/geometryshader/test.frag.qsb b/tests/manual/rhi/geometryshader/test.frag.qsb
index ab1aa3d02e..b6a5877b95 100644
--- a/tests/manual/rhi/geometryshader/test.frag.qsb
+++ b/tests/manual/rhi/geometryshader/test.frag.qsb
diff --git a/tests/manual/rhi/geometryshader/test.geom.qsb b/tests/manual/rhi/geometryshader/test.geom.qsb
index 72ef3bc075..9aa4a0cf98 100644
--- a/tests/manual/rhi/geometryshader/test.geom.qsb
+++ b/tests/manual/rhi/geometryshader/test.geom.qsb
diff --git a/tests/manual/rhi/geometryshader/test.vert.qsb b/tests/manual/rhi/geometryshader/test.vert.qsb
index e317e297cf..7238d8037b 100644
--- a/tests/manual/rhi/geometryshader/test.vert.qsb
+++ b/tests/manual/rhi/geometryshader/test.vert.qsb
diff --git a/tests/manual/rhi/geometryshader/test_geom.hlsl b/tests/manual/rhi/geometryshader/test_geom.hlsl
new file mode 100644
index 0000000000..e58659252b
--- /dev/null
+++ b/tests/manual/rhi/geometryshader/test_geom.hlsl
@@ -0,0 +1,26 @@
+struct VertexOutput
+{
+    float4 position : SV_Position;
+};
+
+struct PixelInput
+{
+    float4 position : SV_POSITION;
+};
+
+cbuffer buf : register(b0)
+{
+    float radius : packoffset(c0);
+};
+
+[maxvertexcount(7)]
+void main(point VertexOutput input[1], inout LineStream<PixelInput> OutputStream)
+{
+    PixelInput output;
+    for (int i = 0; i < 7; ++i) {
+        float theta = float(i) / 6.0f * 2.0 * 3.14159265;
+        output.position = input[0].position;
+        output.position.xy += radius * float2(cos(theta), sin(theta));
+        OutputStream.Append(output);
+    }
+}
diff --git a/tests/manual/rhi/tessellation/buildshaders.bat b/tests/manual/rhi/tessellation/buildshaders.bat
index c44916067d..c9afe1b178 100644
--- a/tests/manual/rhi/tessellation/buildshaders.bat
+++ b/tests/manual/rhi/tessellation/buildshaders.bat
@@ -1,4 +1,6 @@
-qsb --glsl 320es,410 test.vert -o test.vert.qsb
+qsb --glsl 320es,410 --hlsl 50 test.vert -o test.vert.qsb
 qsb --glsl 320es,410 test.tesc -o test.tesc.qsb
+qsb -r hlsl,50,test_hull.hlsl test.tesc.qsb
 qsb --glsl 320es,410 test.tese -o test.tese.qsb
-qsb --glsl 320es,410 test.frag -o test.frag.qsb
+qsb -r hlsl,50,test_domain.hlsl test.tese.qsb
+qsb --glsl 320es,410 --hlsl 50 test.frag -o test.frag.qsb
diff --git a/tests/manual/rhi/tessellation/test.frag.qsb b/tests/manual/rhi/tessellation/test.frag.qsb
index d680373221..4ec03e5700 100644
--- a/tests/manual/rhi/tessellation/test.frag.qsb
+++ b/tests/manual/rhi/tessellation/test.frag.qsb
diff --git a/tests/manual/rhi/tessellation/test.tesc.qsb b/tests/manual/rhi/tessellation/test.tesc.qsb
index fc9dfbaea1..57451d7c08 100644
--- a/tests/manual/rhi/tessellation/test.tesc.qsb
+++ b/tests/manual/rhi/tessellation/test.tesc.qsb
diff --git a/tests/manual/rhi/tessellation/test.tese.qsb b/tests/manual/rhi/tessellation/test.tese.qsb
index fe99cd5001..4ca3c35e92 100644
--- a/tests/manual/rhi/tessellation/test.tese.qsb
+++ b/tests/manual/rhi/tessellation/test.tese.qsb
diff --git a/tests/manual/rhi/tessellation/test.vert.qsb b/tests/manual/rhi/tessellation/test.vert.qsb
index 18f2e7b297..ac261d2b41 100644
--- a/tests/manual/rhi/tessellation/test.vert.qsb
+++ b/tests/manual/rhi/tessellation/test.vert.qsb
diff --git a/tests/manual/rhi/tessellation/test_domain.hlsl b/tests/manual/rhi/tessellation/test_domain.hlsl
new file mode 100644
index 0000000000..a3de658c0e
--- /dev/null
+++ b/tests/manual/rhi/tessellation/test_domain.hlsl
@@ -0,0 +1,38 @@
+struct Input
+{
+    float edges[3] : SV_TessFactor;
+    float inside : SV_InsideTessFactor;
+};
+
+struct PatchInput
+{
+    float3 position : POSITION;
+    float3 color : COLOR;
+};
+
+struct PixelInput
+{
+    float3 color : TEXCOORD0;
+    float4 position : SV_POSITION;
+};
+
+cbuffer buf : register(b0)
+{
+    row_major float4x4 mvp : packoffset(c0);
+    float time : packoffset(c4);
+    float amplitude : packoffset(c4.y);
+};
+
+[domain("tri")]
+PixelInput main(Input input, float3 uvwCoord : SV_DomainLocation, const OutputPatch<PatchInput, 3> patch)
+{
+    PixelInput output;
+
+    float3 vertexPosition = uvwCoord.x * patch[0].position + uvwCoord.y * patch[1].position + uvwCoord.z * patch[2].position;
+    output.position = mul(float4(vertexPosition, 1.0f), mvp);
+    output.position.x += sin(time + output.position.y) * amplitude;
+
+    output.color = uvwCoord.x * patch[0].color + uvwCoord.y * patch[1].color + uvwCoord.z * patch[2].color;
+
+    return output;
+}
diff --git a/tests/manual/rhi/tessellation/test_hull.hlsl b/tests/manual/rhi/tessellation/test_hull.hlsl
new file mode 100644
index 0000000000..3d09307159
--- /dev/null
+++ b/tests/manual/rhi/tessellation/test_hull.hlsl
@@ -0,0 +1,40 @@
+struct Input
+{
+    float3 color : TEXCOORD0;
+    float4 position : SV_Position;
+};
+
+struct Output
+{
+    float3 position : POSITION;
+    float3 color : COLOR;
+};
+
+struct ConstantData
+{
+    float edges[3] : SV_TessFactor;
+    float inside : SV_InsideTessFactor;
+};
+
+ConstantData patchConstFunc(InputPatch<Input, 3> ip, uint PatchID : SV_PrimitiveID )
+{
+    ConstantData d;
+    d.edges[0] = 4.0;
+    d.edges[1] = 4.0;
+    d.edges[2] = 4.0;
+    d.inside = 4.0;
+    return d;
+}
+
+[domain("tri")]
+[partitioning("integer")]
+[outputtopology("triangle_cw")]
+[outputcontrolpoints(3)]
+[patchconstantfunc("patchConstFunc")]
+Output main(InputPatch<Input, 3> patch, uint pointId : SV_OutputControlPointID, uint patchId : SV_PrimitiveID)
+{
+    Output output;
+    output.position = patch[pointId].position;
+    output.color = patch[pointId].color;
+    return output;
+}