rhi: Add the basic infrastructure for geometry shader support

.. but this will only be supported on Vulkan, OpenGL 3.2+, and Open GL
ES 3.2+ for the time being.

The situation is:

- Vulkan is working.  qsb accepts .geom files already, and QShader has
  existing geometry shader support.

- OpenGL 3.2 and OpenGL ES 3.2 are working.

- D3D11 is not working.  D3D11 supports geometry shaders, but SPIRV-
  Cross does not support translating geometry shaders to HLSL.

- Metal is not working.  Metal does not directly support geometry
  shaders.

Change-Id: Ieb7c44c58b8be5f2e2197bf5133cf6847e6c132d
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

9 files changed, 68 insertions, 7 deletions

diff --git a/src/gui/rhi/qrhi.cpp b/src/gui/rhi/qrhi.cpp
index d5a9ef6515..571c5e95b4 100644
--- a/src/gui/rhi/qrhi.cpp
+++ b/src/gui/rhi/qrhi.cpp
@@ -711,6 +711,15 @@ Q_LOGGING_CATEGORY(QRHI_LOG_INFO, "qt.rhi.general")
     the way hull shaders are structured, whereas Metal uses a somewhat
     different tessellation pipeline than others), and therefore no guarantees
     can be given for a universal solution for now.
+
+    \value GeometryShader Indicates that the geometry shader stage is
+    supported. When supported, a geometry shader can be specified in the
+    QRhiShaderStage list. \b{Geometry Shaders are considered an experimental
+    feature in QRhi and can only be expected to be supported with Vulkan,
+    OpenGL (3.2+) and OpenGL ES (3.2+) for the time being}, assuming the
+    implementation reports it as supported at run time. Geometry shaders have
+    portability issues between APIs, and therefore no guarantees can be given
+    for a universal solution for now.
  */
 
 /*!
@@ -1470,6 +1479,9 @@ QDebug operator<<(QDebug dbg, const QRhiVertexInputLayout &v)
 
     \value Compute Compute stage. Must be used only when the QRhi::Compute
     feature is supported.
+
+    \value Geometry Geometry stage. Must be used only when the
+    QRhi::GeometryShader feature is supported.
  */
 
 /*!
@@ -3269,6 +3281,7 @@ void QRhiImplementation::updateLayoutDesc(QRhiShaderResourceBindings *srb)
     \value TessellationEvaluationStage Tessellation evaluation (domain shader) stage
     \value FragmentStage Fragment (pixel shader) stage
     \value ComputeStage Compute stage
+    \value GeometryStage Geometry stage
  */
 
 /*!
@@ -7454,6 +7467,8 @@ QRhiPassResourceTracker::BufferStage QRhiPassResourceTracker::toPassTrackerBuffe
         return QRhiPassResourceTracker::BufFragmentStage;
     if (stages.testFlag(QRhiShaderResourceBinding::ComputeStage))
         return QRhiPassResourceTracker::BufComputeStage;
+    if (stages.testFlag(QRhiShaderResourceBinding::GeometryStage))
+        return QRhiPassResourceTracker::BufGeometryStage;
 
     Q_UNREACHABLE();
     return QRhiPassResourceTracker::BufVertexStage;
@@ -7472,6 +7487,8 @@ QRhiPassResourceTracker::TextureStage QRhiPassResourceTracker::toPassTrackerText
         return QRhiPassResourceTracker::TexFragmentStage;
     if (stages.testFlag(QRhiShaderResourceBinding::ComputeStage))
         return QRhiPassResourceTracker::TexComputeStage;
+    if (stages.testFlag(QRhiShaderResourceBinding::GeometryStage))
+        return QRhiPassResourceTracker::TexGeometryStage;
 
     Q_UNREACHABLE();
     return QRhiPassResourceTracker::TexVertexStage;
diff --git a/src/gui/rhi/qrhi_p.h b/src/gui/rhi/qrhi_p.h
index b0cf9e3469..972aed6fd8 100644
--- a/src/gui/rhi/qrhi_p.h
+++ b/src/gui/rhi/qrhi_p.h
@@ -297,6 +297,7 @@ public:
         Vertex,
         TessellationControl,
         TessellationEvaluation,
+        Geometry,
         Fragment,
         Compute
     };
@@ -351,8 +352,9 @@ public:
         VertexStage = 1 << 0,
         TessellationControlStage = 1 << 1,
         TessellationEvaluationStage = 1 << 2,
-        FragmentStage = 1 << 3,
-        ComputeStage = 1 << 4
+        GeometryStage = 1 << 3,
+        FragmentStage = 1 << 4,
+        ComputeStage = 1 << 5
     };
     Q_DECLARE_FLAGS(StageFlags, StageFlag)
 
@@ -1673,7 +1675,8 @@ public:
         ThreeDimensionalTextures,
         RenderTo3DTextureSlice,
         TextureArrays,
-        Tessellation
+        Tessellation,
+        GeometryShader
     };
 
     enum BeginFrameFlag {
diff --git a/src/gui/rhi/qrhi_p_p.h b/src/gui/rhi/qrhi_p_p.h
index c13b49ce3a..93cbc5ed35 100644
--- a/src/gui/rhi/qrhi_p_p.h
+++ b/src/gui/rhi/qrhi_p_p.h
@@ -613,7 +613,8 @@ public:
         BufTCStage,
         BufTEStage,
         BufFragmentStage,
-        BufComputeStage
+        BufComputeStage,
+        BufGeometryStage
     };
 
     enum BufferAccess {
@@ -635,7 +636,8 @@ public:
         TexFragmentStage,
         TexColorOutputStage,
         TexDepthOutputStage,
-        TexComputeStage
+        TexComputeStage,
+        TexGeometryStage
     };
 
     enum TextureAccess {
diff --git a/src/gui/rhi/qrhid3d11.cpp b/src/gui/rhi/qrhid3d11.cpp
index a4ad0d0c93..28c42a293d 100644
--- a/src/gui/rhi/qrhid3d11.cpp
+++ b/src/gui/rhi/qrhid3d11.cpp
@@ -565,6 +565,8 @@ bool QRhiD3D11::isFeatureSupported(QRhi::Feature feature) const
         return true;
     case QRhi::Tessellation:
         return false;
+    case QRhi::GeometryShader:
+        return false;
     default:
         Q_UNREACHABLE();
         return false;
diff --git a/src/gui/rhi/qrhigles2.cpp b/src/gui/rhi/qrhigles2.cpp
index 21be9c5511..e7a2382497 100644
--- a/src/gui/rhi/qrhigles2.cpp
+++ b/src/gui/rhi/qrhigles2.cpp
@@ -466,6 +466,10 @@ QT_BEGIN_NAMESPACE
 #define GL_PATCHES                        0x000E
 #endif
 
+#ifndef GL_GEOMETRY_SHADER
+#define GL_GEOMETRY_SHADER                0x8DD9
+#endif
+
 /*!
     Constructs a new QRhiGles2InitParams.
 
@@ -867,6 +871,11 @@ bool QRhiGles2::create(QRhi::Flags flags)
     else
         caps.tessellation = caps.ctxMajor >= 4; // 4.0
 
+    if (caps.gles)
+        caps.geometryShader = caps.ctxMajor > 3 || (caps.ctxMajor == 3 && caps.ctxMinor >= 2); // ES 3.2
+    else
+        caps.geometryShader = caps.ctxMajor > 3 || (caps.ctxMajor == 3 && caps.ctxMinor >= 2); // 3.2
+
     if (caps.ctxMajor >= 3) { // 3.0 or ES 3.0
         GLint maxArraySize = 0;
         f->glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &maxArraySize);
@@ -1266,6 +1275,8 @@ bool QRhiGles2::isFeatureSupported(QRhi::Feature feature) const
         return caps.maxTextureArraySize > 0;
     case QRhi::Tessellation:
         return caps.tessellation;
+    case QRhi::GeometryShader:
+        return caps.geometryShader;
     default:
         Q_UNREACHABLE();
         return false;
@@ -4238,6 +4249,8 @@ static inline GLenum toGlShaderType(QRhiShaderStage::Type type)
         return GL_TESS_CONTROL_SHADER;
     case QRhiShaderStage::TessellationEvaluation:
         return GL_TESS_EVALUATION_SHADER;
+    case QRhiShaderStage::Geometry:
+        return GL_GEOMETRY_SHADER;
     case QRhiShaderStage::Fragment:
         return GL_FRAGMENT_SHADER;
     case QRhiShaderStage::Compute:
@@ -4519,6 +4532,8 @@ static inline QShader::Stage toShaderStage(QRhiShaderStage::Type type)
         return QShader::TessellationControlStage;
     case QRhiShaderStage::TessellationEvaluation:
         return QShader::TessellationEvaluationStage;
+    case QRhiShaderStage::Geometry:
+        return QShader::GeometryStage;
     case QRhiShaderStage::Fragment:
         return QShader::FragmentStage;
     case QRhiShaderStage::Compute:
@@ -5447,6 +5462,7 @@ static inline bool isGraphicsStage(const QRhiShaderStage &shaderStage)
     return t == QRhiShaderStage::Vertex
             || t == QRhiShaderStage::TessellationControl
             || t == QRhiShaderStage::TessellationEvaluation
+            || t == QRhiShaderStage::Geometry
             || t == QRhiShaderStage::Fragment;
 }
 
@@ -5469,8 +5485,9 @@ bool QGles2GraphicsPipeline::create()
 
     enum {
         VtxIdx = 0,
-        TEIdx,
         TCIdx,
+        TEIdx,
+        GeomIdx,
         FragIdx,
         LastIdx
     };
@@ -5482,6 +5499,8 @@ bool QGles2GraphicsPipeline::create()
             return TCIdx;
         case QRhiShaderStage::TessellationEvaluation:
             return TEIdx;
+        case QRhiShaderStage::Geometry:
+            return GeomIdx;
         case QRhiShaderStage::Fragment:
             return FragIdx;
         default:
diff --git a/src/gui/rhi/qrhigles2_p_p.h b/src/gui/rhi/qrhigles2_p_p.h
index ad4477d215..3af4178a24 100644
--- a/src/gui/rhi/qrhigles2_p_p.h
+++ b/src/gui/rhi/qrhigles2_p_p.h
@@ -968,7 +968,8 @@ public:
               screenSpaceDerivatives(false),
               programBinary(false),
               texture3D(false),
-              tessellation(false)
+              tessellation(false),
+              geometryShader(false)
         { }
         int ctxMajor;
         int ctxMinor;
@@ -1018,6 +1019,7 @@ public:
         uint programBinary : 1;
         uint texture3D : 1;
         uint tessellation : 1;
+        uint geometryShader : 1;
     } caps;
     QGles2SwapChain *currentSwapChain = nullptr;
     QSet<GLint> supportedCompressedFormats;
diff --git a/src/gui/rhi/qrhimetal.mm b/src/gui/rhi/qrhimetal.mm
index 9f35718838..3d53e8ab2d 100644
--- a/src/gui/rhi/qrhimetal.mm
+++ b/src/gui/rhi/qrhimetal.mm
@@ -620,6 +620,8 @@ bool QRhiMetal::isFeatureSupported(QRhi::Feature feature) const
         return true;
     case QRhi::Tessellation:
         return false;
+    case QRhi::GeometryShader:
+        return false;
     default:
         Q_UNREACHABLE();
         return false;
diff --git a/src/gui/rhi/qrhivulkan.cpp b/src/gui/rhi/qrhivulkan.cpp
index 45306bb266..a99ab3a7f0 100644
--- a/src/gui/rhi/qrhivulkan.cpp
+++ b/src/gui/rhi/qrhivulkan.cpp
@@ -633,6 +633,8 @@ bool QRhiVulkan::create(QRhi::Flags flags)
             features.textureCompressionASTC_LDR = VK_TRUE;
         if (physDevFeatures.textureCompressionBC)
             features.textureCompressionBC = VK_TRUE;
+        if (physDevFeatures.geometryShader)
+            features.geometryShader = VK_TRUE;
         devInfo.pEnabledFeatures = &features;
 
         VkResult err = f->vkCreateDevice(physDev, &devInfo, nullptr, &dev);
@@ -696,6 +698,7 @@ bool QRhiVulkan::create(QRhi::Flags flags)
     caps.texture3DSliceAs2D = caps.vulkan11OrHigher;
 
     caps.tessellation = physDevFeatures.tessellationShader;
+    caps.geometryShader = physDevFeatures.geometryShader;
 
     if (!importedAllocator) {
         VmaVulkanFunctions afuncs;
@@ -3987,6 +3990,8 @@ static inline VkPipelineStageFlags toVkPipelineStage(QRhiPassResourceTracker::Bu
         return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
     case QRhiPassResourceTracker::BufComputeStage:
         return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
+    case QRhiPassResourceTracker::BufGeometryStage:
+        return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
     default:
         Q_UNREACHABLE();
         break;
@@ -4061,6 +4066,8 @@ static inline VkPipelineStageFlags toVkPipelineStage(QRhiPassResourceTracker::Te
         return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
     case QRhiPassResourceTracker::TexComputeStage:
         return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
+    case QRhiPassResourceTracker::TexGeometryStage:
+        return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
     default:
         Q_UNREACHABLE();
         break;
@@ -4317,6 +4324,8 @@ bool QRhiVulkan::isFeatureSupported(QRhi::Feature feature) const
         return true;
     case QRhi::Tessellation:
         return caps.tessellation;
+    case QRhi::GeometryShader:
+        return caps.geometryShader;
     default:
         Q_UNREACHABLE();
         return false;
@@ -5267,6 +5276,8 @@ static inline VkShaderStageFlagBits toVkShaderStage(QRhiShaderStage::Type type)
         return VK_SHADER_STAGE_FRAGMENT_BIT;
     case QRhiShaderStage::Compute:
         return VK_SHADER_STAGE_COMPUTE_BIT;
+    case QRhiShaderStage::Geometry:
+        return VK_SHADER_STAGE_GEOMETRY_BIT;
     default:
         Q_UNREACHABLE();
         return VK_SHADER_STAGE_VERTEX_BIT;
@@ -5559,6 +5570,8 @@ static inline VkShaderStageFlags toVkShaderStageFlags(QRhiShaderResourceBinding:
         s |= VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT;
     if (stage.testFlag(QRhiShaderResourceBinding::TessellationEvaluationStage))
         s |= VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT;
+    if (stage.testFlag(QRhiShaderResourceBinding::GeometryStage))
+        s |= VK_SHADER_STAGE_GEOMETRY_BIT;
     return VkShaderStageFlags(s);
 }
 
diff --git a/src/gui/rhi/qrhivulkan_p_p.h b/src/gui/rhi/qrhivulkan_p_p.h
index e48bcff3fa..f916d7d2b5 100644
--- a/src/gui/rhi/qrhivulkan_p_p.h
+++ b/src/gui/rhi/qrhivulkan_p_p.h
@@ -891,6 +891,7 @@ public:
         bool texture3DSliceAs2D = false;
         bool tessellation = false;
         bool vulkan11OrHigher = false;
+        bool geometryShader = false;
     } caps;
 
     VkPipelineCache pipelineCache = VK_NULL_HANDLE;