rhi: d3d: Allow compiling source shaders with debug at run time

Relevant for Qt Quick 3D. As in many cases Quick3D will rely on runtime
generated shader code, the translated HLSL will be compiled via
D3DCompile() at run time. To make such shaders debuggable, the necessary
flag (D3DCOMPILE_DEBUG) should be requestable somehow.

Change-Id: I4d5c3b57bf58df8d46556eebb5cf6fb75e9d0afe
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

3 files changed, 56 insertions, 6 deletions

diff --git a/src/gui/rhi/qrhi.cpp b/src/gui/rhi/qrhi.cpp
index c82faf2566..7ebac6d1bd 100644
--- a/src/gui/rhi/qrhi.cpp
+++ b/src/gui/rhi/qrhi.cpp
@@ -3463,7 +3463,8 @@ QDebug operator<<(QDebug dbg, const QRhiShaderResourceBindings &srb)
 /*!
     \enum QRhiGraphicsPipeline::Flag
 
-    Flag values for describing the dynamic state of the pipeline. The viewport is always dynamic.
+    Flag values for describing the dynamic state of the pipeline, and other
+    options. The viewport is always dynamic.
 
     \value UsesBlendConstants Indicates that a blend color constant will be set
     via QRhiCommandBuffer::setBlendConstants()
@@ -3473,6 +3474,22 @@ QDebug operator<<(QDebug dbg, const QRhiShaderResourceBindings &srb)
 
     \value UsesScissor Indicates that a scissor rectangle will be set via
     QRhiCommandBuffer::setScissor()
+
+    \value CompileShadersWithDebugInfo Requests compiling shaders with debug
+    information enabled. This is relevant only when runtime shader compilation
+    from source code is involved, and only when the underlying infrastructure
+    supports this. With concrete examples, this is not relevant with Vulkan and
+    SPIR-V, because the GLSL-to-SPIR-V compilation does not happen at run
+    time. On the other hand, consider Direct3D and HLSL, where there are
+    multiple options: when the QShader packages ship with pre-compiled bytecode
+    (\c DXBC), debug information is to be requested through the tool that
+    generates the \c{.qsb} file, similarly to the case of Vulkan and
+    SPIR-V. However, when having HLSL source code in the pre- or
+    runtime-generated QShader packages, the first phase of compilation (HLSL
+    source to intermediate format) happens at run time too, with this flag taken
+    into account. Debug information is relevant in particular with tools like
+    RenderDoc since it allows seeing the original source code when investigating
+    the pipeline and when performing vertex or fragment shader debugging.
  */
 
 /*!
@@ -3963,6 +3980,16 @@ QRhiResource::Type QRhiSwapChain::resourceType() const
  */
 
 /*!
+    \enum QRhiComputePipeline::Flag
+
+    Flag values for describing pipeline options.
+
+    \value CompileShadersWithDebugInfo Requests compiling shaders with debug
+    information enabled, when applicable. See
+    QRhiGraphicsPipeline::CompileShadersWithDebugInfo for more information.
+ */
+
+/*!
     \return the resource type.
  */
 QRhiResource::Type QRhiComputePipeline::resourceType() const
diff --git a/src/gui/rhi/qrhi_p.h b/src/gui/rhi/qrhi_p.h
index dd8ab862e4..7d719fd218 100644
--- a/src/gui/rhi/qrhi_p.h
+++ b/src/gui/rhi/qrhi_p.h
@@ -1024,7 +1024,8 @@ public:
     enum Flag {
         UsesBlendConstants = 1 << 0,
         UsesStencilRef = 1 << 1,
-        UsesScissor = 1 << 2
+        UsesScissor = 1 << 2,
+        CompileShadersWithDebugInfo = 1 << 3
     };
     Q_DECLARE_FLAGS(Flags, Flag)
 
@@ -1289,9 +1290,17 @@ Q_DECLARE_OPERATORS_FOR_FLAGS(QRhiSwapChain::Flags)
 class Q_GUI_EXPORT QRhiComputePipeline : public QRhiResource
 {
 public:
+    enum Flag {
+        CompileShadersWithDebugInfo = 1 << 0
+    };
+    Q_DECLARE_FLAGS(Flags, Flag)
+
     QRhiResource::Type resourceType() const override;
     virtual bool create() = 0;
 
+    Flags flags() const { return m_flags; }
+    void setFlags(Flags f) { m_flags = f; }
+
     QRhiShaderStage shaderStage() const { return m_shaderStage; }
     void setShaderStage(const QRhiShaderStage &stage) { m_shaderStage = stage; }
 
@@ -1300,10 +1309,13 @@ public:
 
 protected:
     QRhiComputePipeline(QRhiImplementation *rhi);
+    Flags m_flags;
     QRhiShaderStage m_shaderStage;
     QRhiShaderResourceBindings *m_shaderResourceBindings = nullptr;
 };
 
+Q_DECLARE_OPERATORS_FOR_FLAGS(QRhiComputePipeline::Flags)
+
 class Q_GUI_EXPORT QRhiCommandBuffer : public QRhiResource
 {
 public:
diff --git a/src/gui/rhi/qrhid3d11.cpp b/src/gui/rhi/qrhid3d11.cpp
index f105d01e4f..71b8e48da0 100644
--- a/src/gui/rhi/qrhid3d11.cpp
+++ b/src/gui/rhi/qrhid3d11.cpp
@@ -3698,7 +3698,8 @@ static pD3DCompile resolveD3DCompile()
     return nullptr;
 }
 
-static QByteArray compileHlslShaderSource(const QShader &shader, QShader::Variant shaderVariant, QString *error, QShaderKey *usedShaderKey)
+static QByteArray compileHlslShaderSource(const QShader &shader, QShader::Variant shaderVariant, UINT flags,
+                                          QString *error, QShaderKey *usedShaderKey)
 {
     QShaderKey key = { QShader::DxbcShader, 50, shaderVariant };
     QShaderCode dxbc = shader.shader(key);
@@ -3750,7 +3751,7 @@ static QByteArray compileHlslShaderSource(const QShader &shader, QShader::Varian
     ID3DBlob *errors = nullptr;
     HRESULT hr = d3dCompile(hlslSource.shader().constData(), SIZE_T(hlslSource.shader().size()),
                             nullptr, nullptr, nullptr,
-                            hlslSource.entryPoint().constData(), target, 0, 0, &bytecode, &errors);
+                            hlslSource.entryPoint().constData(), target, flags, 0, &bytecode, &errors);
     if (FAILED(hr) || !bytecode) {
         qWarning("HLSL shader compilation failed: 0x%x", uint(hr));
         if (errors) {
@@ -3871,7 +3872,12 @@ bool QD3D11GraphicsPipeline::create()
         } else {
             QString error;
             QShaderKey shaderKey;
-            const QByteArray bytecode = compileHlslShaderSource(shaderStage.shader(), shaderStage.shaderVariant(), &error, &shaderKey);
+            UINT compileFlags = 0;
+            if (m_flags.testFlag(CompileShadersWithDebugInfo))
+                compileFlags |= D3DCOMPILE_DEBUG;
+
+            const QByteArray bytecode = compileHlslShaderSource(shaderStage.shader(), shaderStage.shaderVariant(), compileFlags,
+                                                                &error, &shaderKey);
             if (bytecode.isEmpty()) {
                 qWarning("HLSL shader compilation failed: %s", qPrintable(error));
                 return false;
@@ -3987,7 +3993,12 @@ bool QD3D11ComputePipeline::create()
     } else {
         QString error;
         QShaderKey shaderKey;
-        const QByteArray bytecode = compileHlslShaderSource(m_shaderStage.shader(), m_shaderStage.shaderVariant(), &error, &shaderKey);
+        UINT compileFlags = 0;
+        if (m_flags.testFlag(CompileShadersWithDebugInfo))
+            compileFlags |= D3DCOMPILE_DEBUG;
+
+        const QByteArray bytecode = compileHlslShaderSource(m_shaderStage.shader(), m_shaderStage.shaderVariant(), compileFlags,
+                                                            &error, &shaderKey);
         if (bytecode.isEmpty()) {
             qWarning("HLSL compute shader compilation failed: %s", qPrintable(error));
             return false;