rhi: Add support for polygon fill mode

Support for Polygon Mode (Triangle Fill Mode in Metal, Fill Mode in D3D)
in the RHI graphics pipeline.

Options are Fill and Line

Status:
    OpenGL - ok
    Vulkan - ok
    Metal - ok
    D3D11 - ok
    OpenGL ES - does not support glPolygonMode.

Change-Id: I20b7ef416624700c3dc8d1cbe6474f4ca3889db8
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

8 files changed, 103 insertions, 1 deletions

diff --git a/src/gui/rhi/qrhi.cpp b/src/gui/rhi/qrhi.cpp
index f583160f3c..d5a9ef6515 100644
--- a/src/gui/rhi/qrhi.cpp
+++ b/src/gui/rhi/qrhi.cpp
@@ -4256,6 +4256,20 @@ QDebug operator<<(QDebug dbg, const QRhiShaderResourceBindings &srb)
  */
 
 /*!
+    \enum QRhiGraphicsPipeline::PolygonMode
+    \brief Specifies the polygon rasterization mode
+
+    Polygon Mode (Triangle Fill Mode in Metal, Fill Mode in D3D) specifies
+    the fill mode used when rasterizing polygons.  Polygons may be drawn as
+    solids (Fill), or as a wire mesh (Line).
+
+    \note OpenGL ES does not support Polygon Mode
+
+    \value Fill The interior of the polygon is filled (default)
+    \value Line Boundary edges of the polygon are drawn as line segments.
+ */
+
+/*!
     \class QRhiGraphicsPipeline::TargetBlend
     \internal
     \inmodule QtGui
diff --git a/src/gui/rhi/qrhi_p.h b/src/gui/rhi/qrhi_p.h
index 8c38b6383d..b0cf9e3469 100644
--- a/src/gui/rhi/qrhi_p.h
+++ b/src/gui/rhi/qrhi_p.h
@@ -1223,6 +1223,11 @@ public:
         CompareOp compareOp = Always;
     };
 
+    enum PolygonMode {
+        Fill,
+        Line
+    };
+
     QRhiResource::Type resourceType() const override;
 
     Flags flags() const { return m_flags; }
@@ -1305,6 +1310,9 @@ public:
     int patchControlPointCount() const { return m_patchControlPointCount; }
     void setPatchControlPointCount(int count) { m_patchControlPointCount = count; }
 
+    PolygonMode polygonMode() const {return m_polygonMode; }
+    void setPolygonMode(PolygonMode mode) {m_polygonMode = mode; }
+
     virtual bool create() = 0;
 
 protected:
@@ -1327,6 +1335,7 @@ protected:
     int m_depthBias = 0;
     float m_slopeScaledDepthBias = 0.0f;
     int m_patchControlPointCount = 3;
+    PolygonMode m_polygonMode = Fill;
     QVarLengthArray<QRhiShaderStage, 4> m_shaderStages;
     QRhiVertexInputLayout m_vertexInputLayout;
     QRhiShaderResourceBindings *m_shaderResourceBindings = nullptr;
diff --git a/src/gui/rhi/qrhid3d11.cpp b/src/gui/rhi/qrhid3d11.cpp
index fc6fd59f4f..a4ad0d0c93 100644
--- a/src/gui/rhi/qrhid3d11.cpp
+++ b/src/gui/rhi/qrhid3d11.cpp
@@ -3783,6 +3783,19 @@ static inline D3D11_CULL_MODE toD3DCullMode(QRhiGraphicsPipeline::CullMode c)
     }
 }
 
+static inline D3D11_FILL_MODE toD3DFillMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::Fill:
+        return D3D11_FILL_SOLID;
+    case QRhiGraphicsPipeline::Line:
+        return D3D11_FILL_WIREFRAME;
+    default:
+        Q_UNREACHABLE();
+        return D3D11_FILL_SOLID;
+    }
+}
+
 static inline D3D11_COMPARISON_FUNC toD3DCompareOp(QRhiGraphicsPipeline::CompareOp op)
 {
     switch (op) {
@@ -4072,7 +4085,7 @@ bool QD3D11GraphicsPipeline::create()
 
     D3D11_RASTERIZER_DESC rastDesc;
     memset(&rastDesc, 0, sizeof(rastDesc));
-    rastDesc.FillMode = D3D11_FILL_SOLID;
+    rastDesc.FillMode = toD3DFillMode(m_polygonMode);
     rastDesc.CullMode = toD3DCullMode(m_cullMode);
     rastDesc.FrontCounterClockwise = m_frontFace == CCW;
     rastDesc.DepthBias = m_depthBias;
diff --git a/src/gui/rhi/qrhigles2.cpp b/src/gui/rhi/qrhigles2.cpp
index 027fd7c5f8..21be9c5511 100644
--- a/src/gui/rhi/qrhigles2.cpp
+++ b/src/gui/rhi/qrhigles2.cpp
@@ -454,6 +454,14 @@ QT_BEGIN_NAMESPACE
 #define GL_PATCH_VERTICES                 0x8E72
 #endif
 
+#ifndef GL_LINE
+#define GL_LINE                           0x1B01
+#endif
+
+#ifndef GL_FILL
+#define GL_FILL                           0x1B02
+#endif
+
 #ifndef GL_PATCHES
 #define GL_PATCHES                        0x000E
 #endif
@@ -661,6 +669,8 @@ bool QRhiGles2::create(QRhi::Flags flags)
         return false;
 
     f = static_cast<QOpenGLExtensions *>(ctx->extraFunctions());
+    glPolygonMode = reinterpret_cast<void(QOPENGLF_APIENTRYP)(GLenum, GLenum)>(
+            ctx->getProcAddress(QByteArrayLiteral("glPolygonMode")));
 
     const char *vendor = reinterpret_cast<const char *>(f->glGetString(GL_VENDOR));
     const char *renderer = reinterpret_cast<const char *>(f->glGetString(GL_RENDERER));
@@ -2506,6 +2516,19 @@ static inline GLenum toGlStencilOp(QRhiGraphicsPipeline::StencilOp op)
     }
 }
 
+static inline GLenum toGlPolygonMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::PolygonMode::Fill:
+        return GL_FILL;
+    case QRhiGraphicsPipeline::PolygonMode::Line:
+        return GL_LINE;
+    default:
+        Q_UNREACHABLE();
+        return GL_FILL;
+    }
+}
+
 static inline GLenum toGlMinFilter(QRhiSampler::Filter f, QRhiSampler::Filter m)
 {
     switch (f) {
@@ -3318,6 +3341,12 @@ void QRhiGles2::executeBindGraphicsPipeline(QGles2CommandBuffer *cbD, QGles2Grap
         f->glFrontFace(frontFace);
     }
 
+    const GLenum polygonMode = toGlPolygonMode(psD->m_polygonMode);
+    if (glPolygonMode && (forceUpdate || polygonMode != state.polygonMode)) {
+        state.polygonMode = polygonMode;
+        glPolygonMode(GL_FRONT_AND_BACK, polygonMode);
+    }
+
     if (!psD->m_targetBlends.isEmpty()) {
         // We do not have MRT support here, meaning all targets use the blend
         // params from the first one. This is technically incorrect, even if
diff --git a/src/gui/rhi/qrhigles2_p_p.h b/src/gui/rhi/qrhigles2_p_p.h
index ade0ec9748..ad4477d215 100644
--- a/src/gui/rhi/qrhigles2_p_p.h
+++ b/src/gui/rhi/qrhigles2_p_p.h
@@ -603,6 +603,7 @@ struct QGles2CommandBuffer : public QRhiCommandBuffer
         float polyOffsetUnits;
         float lineWidth;
         int cpCount;
+        GLenum polygonMode;
         void reset() { valid = false; }
         struct {
             // not part of QRhiGraphicsPipeline but used by setGraphicsPipeline()
@@ -918,6 +919,7 @@ public:
     QOpenGLContext *maybeShareContext = nullptr;
     mutable bool needsMakeCurrentDueToSwap = false;
     QOpenGLExtensions *f = nullptr;
+    void (QOPENGLF_APIENTRYP glPolygonMode) (GLenum, GLenum) = nullptr;
     uint vao = 0;
     struct Caps {
         Caps()
diff --git a/src/gui/rhi/qrhimetal.mm b/src/gui/rhi/qrhimetal.mm
index 4198306ffc..9f35718838 100644
--- a/src/gui/rhi/qrhimetal.mm
+++ b/src/gui/rhi/qrhimetal.mm
@@ -307,6 +307,7 @@ struct QMetalGraphicsPipelineData
     MTLPrimitiveType primitiveType;
     MTLWinding winding;
     MTLCullMode cullMode;
+    MTLTriangleFillMode triangleFillMode;
     float depthBias;
     float slopeScaledDepthBias;
     QMetalShader vs;
@@ -1064,6 +1065,10 @@ void QRhiMetal::setGraphicsPipeline(QRhiCommandBuffer *cb, QRhiGraphicsPipeline
             [cbD->d->currentRenderPassEncoder setCullMode: psD->d->cullMode];
             cbD->currentCullMode = int(psD->d->cullMode);
         }
+        if (cbD->currentTriangleFillMode == -1 || psD->d->triangleFillMode != uint(cbD->currentTriangleFillMode)) {
+            [cbD->d->currentRenderPassEncoder setTriangleFillMode: psD->d->triangleFillMode];
+            cbD->currentTriangleFillMode = int(psD->d->triangleFillMode);
+        }
         if (cbD->currentFrontFaceWinding == -1 || psD->d->winding != uint(cbD->currentFrontFaceWinding)) {
             [cbD->d->currentRenderPassEncoder setFrontFacingWinding: psD->d->winding];
             cbD->currentFrontFaceWinding = int(psD->d->winding);
@@ -3498,6 +3503,19 @@ static inline MTLCullMode toMetalCullMode(QRhiGraphicsPipeline::CullMode c)
     }
 }
 
+static inline MTLTriangleFillMode toMetalTriangleFillMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::Fill:
+        return MTLTriangleFillModeFill;
+    case QRhiGraphicsPipeline::Line:
+        return MTLTriangleFillModeLines;
+    default:
+        Q_UNREACHABLE();
+        return MTLTriangleFillModeFill;
+    }
+}
+
 id<MTLLibrary> QRhiMetalData::createMetalLib(const QShader &shader, QShader::Variant shaderVariant,
                                              QString *error, QByteArray *entryPoint, QShaderKey *activeKey)
 {
@@ -3754,6 +3772,7 @@ bool QMetalGraphicsPipeline::create()
     d->primitiveType = toMetalPrimitiveType(m_topology);
     d->winding = m_frontFace == CCW ? MTLWindingCounterClockwise : MTLWindingClockwise;
     d->cullMode = toMetalCullMode(m_cullMode);
+    d->triangleFillMode = toMetalTriangleFillMode(m_polygonMode);
     d->depthBias = float(m_depthBias);
     d->slopeScaledDepthBias = m_slopeScaledDepthBias;
 
@@ -3902,6 +3921,7 @@ void QMetalCommandBuffer::resetPerPassCachedState()
     currentIndexOffset = 0;
     currentIndexFormat = QRhiCommandBuffer::IndexUInt16;
     currentCullMode = -1;
+    currentTriangleFillMode = -1;
     currentFrontFaceWinding = -1;
     currentDepthBiasValues = { 0.0f, 0.0f };
 
diff --git a/src/gui/rhi/qrhimetal_p_p.h b/src/gui/rhi/qrhimetal_p_p.h
index 87d533ccaa..5234a152e1 100644
--- a/src/gui/rhi/qrhimetal_p_p.h
+++ b/src/gui/rhi/qrhimetal_p_p.h
@@ -303,6 +303,7 @@ struct QMetalCommandBuffer : public QRhiCommandBuffer
     quint32 currentIndexOffset;
     QRhiCommandBuffer::IndexFormat currentIndexFormat;
     int currentCullMode;
+    int currentTriangleFillMode;
     int currentFrontFaceWinding;
     QPair<float, float> currentDepthBiasValues;
 
diff --git a/src/gui/rhi/qrhivulkan.cpp b/src/gui/rhi/qrhivulkan.cpp
index c27b55f885..45306bb266 100644
--- a/src/gui/rhi/qrhivulkan.cpp
+++ b/src/gui/rhi/qrhivulkan.cpp
@@ -5493,6 +5493,19 @@ static inline VkStencilOp toVkStencilOp(QRhiGraphicsPipeline::StencilOp op)
     }
 }
 
+static inline VkPolygonMode toVkPolygonMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::Fill:
+        return VK_POLYGON_MODE_FILL;
+    case QRhiGraphicsPipeline::Line:
+        return VK_POLYGON_MODE_LINE;
+    default:
+        Q_UNREACHABLE();
+        return VK_POLYGON_MODE_FILL;
+    }
+}
+
 static inline void fillVkStencilOpState(VkStencilOpState *dst, const QRhiGraphicsPipeline::StencilOpState &src)
 {
     dst->failOp = toVkStencilOp(src.failOp);
@@ -7009,6 +7022,7 @@ bool QVkGraphicsPipeline::create()
         rastInfo.depthBiasSlopeFactor = m_slopeScaledDepthBias;
     }
     rastInfo.lineWidth = rhiD->caps.wideLines ? m_lineWidth : 1.0f;
+    rastInfo.polygonMode = toVkPolygonMode(m_polygonMode);
     pipelineInfo.pRasterizationState = &rastInfo;
 
     VkPipelineMultisampleStateCreateInfo msInfo;