D3D12: Support runtime threaded shader compilation

Let's revise our policy of offline/bytecode only shaders. ShaderEffect
benefits greatly from having runtime compilation support for HLSL
source strings, especially when dynamically constructing shader strings.
There is no reason not to support both approaches since we rely on d3dcompiler
for reflection anyhow.

What's more, we can call D3DCompile on a dedicated thread, keeping the
gui responsive while compilation is on-going.

Change-Id: Ie6c02c2aa0ebd0c8371bbf30b3ce6582128c457b
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

3 files changed, 102 insertions, 26 deletions

diff --git a/src/quick/items/qquickgenericshadereffect.cpp b/src/quick/items/qquickgenericshadereffect.cpp
index 47272a2eac..11259a588a 100644
--- a/src/quick/items/qquickgenericshadereffect.cpp
+++ b/src/quick/items/qquickgenericshadereffect.cpp
@@ -61,7 +61,10 @@ QQuickGenericShaderEffect::QQuickGenericShaderEffect(QQuickShaderEffect *item, Q
     , m_mgr(nullptr)
     , m_dirty(0)
 {
+    qRegisterMetaType<QSGGuiThreadShaderEffectManager::ShaderInfo::Type>("ShaderInfo::Type");
     connect(m_item, SIGNAL(windowChanged(QQuickWindow*)), this, SLOT(itemWindowChanged(QQuickWindow*)));
+    for (int i = 0; i < NShader; ++i)
+        m_inProgress[i] = nullptr;
 }
 
 QQuickGenericShaderEffect::~QQuickGenericShaderEffect()
@@ -232,6 +235,10 @@ QSGNode *QQuickGenericShaderEffect::handleUpdatePaintNode(QSGNode *oldNode, QQui
         return nullptr;
     }
 
+    // Do not change anything while a new shader is being reflected or compiled.
+    if (m_inProgress[Vertex] || m_inProgress[Fragment])
+        return node;
+
     // The manager should be already created on the gui thread. Just take that instance.
     QSGGuiThreadShaderEffectManager *mgr = shaderEffectManager();
     if (!mgr) {
@@ -327,6 +334,7 @@ QSGGuiThreadShaderEffectManager *QQuickGenericShaderEffect::shaderEffectManager(
                 connect(m_mgr, SIGNAL(logAndStatusChanged()), m_item, SIGNAL(logChanged()));
                 connect(m_mgr, SIGNAL(logAndStatusChanged()), m_item, SIGNAL(statusChanged()));
                 connect(m_mgr, SIGNAL(textureChanged()), this, SLOT(markGeometryDirtyAndUpdateIfSupportsAtlas()));
+                connect(m_mgr, &QSGGuiThreadShaderEffectManager::shaderCodePrepared, this, &QQuickGenericShaderEffect::shaderCodePrepared);
             }
         } else if (!w) {
             // Wait until itemWindowChanged() gets called. Return null for now.
@@ -377,27 +385,27 @@ void QQuickGenericShaderEffect::disconnectSignals(Shader shaderType)
     }
 }
 
-struct ReflectCache
+struct ShaderInfoCache
 {
     bool contains(const QByteArray &key) const
     {
-        return m_reflectCache.contains(key);
+        return m_shaderInfoCache.contains(key);
     }
 
     QSGGuiThreadShaderEffectManager::ShaderInfo value(const QByteArray &key) const
     {
-        return m_reflectCache.value(key);
+        return m_shaderInfoCache.value(key);
     }
 
     void insert(const QByteArray &key, const QSGGuiThreadShaderEffectManager::ShaderInfo &value)
     {
-        m_reflectCache.insert(key, value);
+        m_shaderInfoCache.insert(key, value);
     }
 
-    QHash<QByteArray, QSGGuiThreadShaderEffectManager::ShaderInfo> m_reflectCache;
+    QHash<QByteArray, QSGGuiThreadShaderEffectManager::ShaderInfo> m_shaderInfoCache;
 };
 
-Q_GLOBAL_STATIC(ReflectCache, reflectCache)
+Q_GLOBAL_STATIC(ShaderInfoCache, shaderInfoCache)
 
 void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray &src)
 {
@@ -413,22 +421,26 @@ void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray
     m_shaders[shaderType].varData.clear();
 
     if (!src.isEmpty()) {
-        // Figure out what input parameters and variables are used in the shader.
-        // For file-based shader source/bytecode this is where the data is pulled
-        // in from the file.
-        if (reflectCache()->contains(src)) {
-            m_shaders[shaderType].shaderInfo = reflectCache()->value(src);
+        if (shaderInfoCache()->contains(src)) {
+            m_shaders[shaderType].shaderInfo = shaderInfoCache()->value(src);
+            m_shaders[shaderType].hasShaderCode = true;
         } else {
-            QSGGuiThreadShaderEffectManager::ShaderInfo shaderInfo;
-            if (!mgr->reflect(src, &shaderInfo)) {
-                qWarning("ShaderEffect: shader reflection failed for %s", src.constData());
-                m_shaders[shaderType].hasShaderCode = false;
-                return;
-            }
-            m_shaders[shaderType].shaderInfo = shaderInfo;
-            reflectCache()->insert(src, shaderInfo);
+            // Each prepareShaderCode call needs its own work area, hence the
+            // dynamic alloc. If there are calls in progress, let those run to
+            // finish, their results can then simply be ignored because
+            // m_inProgress indicates what we care about.
+            m_inProgress[shaderType] = new QSGGuiThreadShaderEffectManager::ShaderInfo;
+            const QSGGuiThreadShaderEffectManager::ShaderInfo::Type typeHint =
+                    shaderType == Vertex ? QSGGuiThreadShaderEffectManager::ShaderInfo::TypeVertex
+                                         : QSGGuiThreadShaderEffectManager::ShaderInfo::TypeFragment;
+            // Figure out what input parameters and variables are used in the
+            // shader. For file-based shader source/bytecode this is where the data
+            // is pulled in from the file. Some backends may choose to do
+            // source->bytecode compilation as well in this step.
+            mgr->prepareShaderCode(typeHint, src, m_inProgress[shaderType]);
+            // the rest is handled in shaderCodePrepared()
+            return;
         }
-        m_shaders[shaderType].hasShaderCode = true;
     } else {
         m_shaders[shaderType].hasShaderCode = false;
         if (shaderType == Fragment) {
@@ -446,6 +458,44 @@ void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray
         }
     }
 
+    updateShaderVars(shaderType);
+}
+
+void QQuickGenericShaderEffect::shaderCodePrepared(bool ok, QSGGuiThreadShaderEffectManager::ShaderInfo::Type typeHint,
+                                                   const QByteArray &src, QSGGuiThreadShaderEffectManager::ShaderInfo *result)
+{
+    const Shader shaderType = typeHint == QSGGuiThreadShaderEffectManager::ShaderInfo::TypeVertex ? Vertex : Fragment;
+
+    // If another call was made to updateShader() for the same shader type in
+    // the meantime then our results are useless, just drop them.
+    if (result != m_inProgress[shaderType]) {
+        delete result;
+        return;
+    }
+
+    m_shaders[shaderType].shaderInfo = *result;
+    delete result;
+    m_inProgress[shaderType] = nullptr;
+
+    if (!ok) {
+        qWarning("ShaderEffect: shader preparation failed for %s\n%s\n", src.constData(), qPrintable(log()));
+        m_shaders[shaderType].hasShaderCode = false;
+        return;
+    }
+
+    m_shaders[shaderType].hasShaderCode = true;
+    shaderInfoCache()->insert(src, m_shaders[shaderType].shaderInfo);
+    updateShaderVars(shaderType);
+}
+
+void QQuickGenericShaderEffect::updateShaderVars(Shader shaderType)
+{
+    QSGGuiThreadShaderEffectManager *mgr = shaderEffectManager();
+    if (!mgr)
+        return;
+
+    const bool texturesSeparate = mgr->hasSeparateSamplerAndTextureObjects();
+
     const int varCount = m_shaders[shaderType].shaderInfo.variables.count();
     m_shaders[shaderType].varData.resize(varCount);
 
diff --git a/src/quick/items/qquickgenericshadereffect_p.h b/src/quick/items/qquickgenericshadereffect_p.h
index ab17a7fb87..5ec83eb60f 100644
--- a/src/quick/items/qquickgenericshadereffect_p.h
+++ b/src/quick/items/qquickgenericshadereffect_p.h
@@ -103,6 +103,8 @@ private slots:
     void markGeometryDirtyAndUpdateIfSupportsAtlas();
     void itemWindowChanged(QQuickWindow *w);
     void backendChanged();
+    void shaderCodePrepared(bool ok, QSGGuiThreadShaderEffectManager::ShaderInfo::Type typeHint,
+                            const QByteArray &src, QSGGuiThreadShaderEffectManager::ShaderInfo *result);
 
 private:
     QSGGuiThreadShaderEffectManager *shaderEffectManager() const;
@@ -113,8 +115,9 @@ private:
 
         NShader
     };
-    void updateShader(Shader which, const QByteArray &src);
-    void disconnectSignals(Shader which);
+    void updateShader(Shader shaderType, const QByteArray &src);
+    void updateShaderVars(Shader shaderType);
+    void disconnectSignals(Shader shaderType);
     bool sourceIsUnique(QQuickItem *source, Shader typeToSkip, int indexToSkip) const;
 
     QQuickShaderEffect *m_item;
@@ -132,6 +135,7 @@ private:
     QSGShaderEffectNode::DirtyShaderFlags m_dirty;
     QSet<int> m_dirtyConstants[NShader];
     QSet<int> m_dirtyTextures[NShader];
+    QSGGuiThreadShaderEffectManager::ShaderInfo *m_inProgress[NShader];
 
     struct SignalMapper {
         SignalMapper() : mapper(nullptr), active(false) { }
diff --git a/src/quick/items/qquickshadereffect.cpp b/src/quick/items/qquickshadereffect.cpp
index f7fc7880ed..4f1a9a28ec 100644
--- a/src/quick/items/qquickshadereffect.cpp
+++ b/src/quick/items/qquickshadereffect.cpp
@@ -209,11 +209,33 @@ QT_BEGIN_NAMESPACE
     it is the textures that map to properties referencing \l Image or
     \l ShaderEffectSource items.
 
-    Unlike with OpenGL, runtime compilation of shader source code may not be
-    supported. Backends for modern APIs are likely to prefer offline
+    Unlike OpenGL, backends for modern APIs will typically prefer offline
     compilation and shipping pre-compiled bytecode with applications instead of
-    inlined shader source strings. To check what is expected at runtime, use the
-    GraphicsInfo.shaderSourceType and GraphicsInfo.shaderCompilationType properties.
+    inlined shader source strings. In this case the string properties for
+    vertex and fragment shaders are treated as URLs referring to local files or
+    files shipped via the Qt resource system.
+
+    To check at runtime what is supported, use the
+    GraphicsInfo.shaderSourceType and GraphicsInfo.shaderCompilationType
+    properties. Note that these are bitmasks, because some backends may support
+    multiple approaches.
+
+    In case of Direct3D 12, both bytecode in files and HLSL source strings are
+    supported. If the vertexShader and fragmentShader properties form a valid
+    URL with the \c file or \c qrc schema, the bytecode is read from the
+    specified file. Otherwise, the string is treated as HLSL source code and is
+    compiled at runtime, assuming Shader Model 5.0 and an entry point of
+    \c{"main"}. This allows dynamically constructing shader strings. However,
+    whenever the shader source code is static, it is strongly recommended to
+    pre-compile to bytecode using the \c fxc tool and refer to these files from
+    QML. This will be a lot more efficient at runtime and allows catching
+    syntax errors in the shaders at compile time.
+
+    Unlike OpenGL, the Direct3D backend is able to perform runtime shader
+    compilation on dedicated threads. This is managed transparently to the
+    applications, and means that ShaderEffect items that contain HLSL source
+    strings do not block the rendering or other parts of the application until
+    the bytecode is ready.
 
     \table 70%
     \row