D3D12: Support runtime threaded shader compilation

Let's revise our policy of offline/bytecode only shaders. ShaderEffect
benefits greatly from having runtime compilation support for HLSL
source strings, especially when dynamically constructing shader strings.
There is no reason not to support both approaches since we rely on d3dcompiler
for reflection anyhow.

What's more, we can call D3DCompile on a dedicated thread, keeping the
gui responsive while compilation is on-going.

Change-Id: Ie6c02c2aa0ebd0c8371bbf30b3ce6582128c457b
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

7 files changed, 195 insertions, 41 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index 45b9de9ece..b54d10aa85 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -552,12 +552,12 @@ QSGRendererInterface::ShaderType QSGD3D12Engine::shaderType() const
 
 QSGRendererInterface::ShaderCompilationTypes QSGD3D12Engine::shaderCompilationType() const
 {
-    return OfflineCompilation;
+    return RuntimeCompilation | OfflineCompilation;
 }
 
 QSGRendererInterface::ShaderSourceTypes QSGD3D12Engine::shaderSourceType() const
 {
-    return ShaderByteCode;
+    return ShaderSourceString | ShaderByteCode;
 }
 
 static inline quint32 alignedSize(quint32 size, quint32 byteAlign)
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp
index f77719f876..8c46e781e9 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp
@@ -41,6 +41,7 @@
 #include "qsgd3d12rendercontext_p.h"
 #include "qsgd3d12texture_p.h"
 #include "qsgd3d12engine_p.h"
+#include <QtCore/qthreadpool.h>
 #include <QtCore/qfile.h>
 #include <QtQml/qqmlfile.h>
 #include <qsgtextureprovider.h>
@@ -777,12 +778,12 @@ bool QSGD3D12GuiThreadShaderEffectManager::hasSeparateSamplerAndTextureObjects()
 
 QString QSGD3D12GuiThreadShaderEffectManager::log() const
 {
-    return QString();
+    return m_log;
 }
 
 QSGGuiThreadShaderEffectManager::Status QSGD3D12GuiThreadShaderEffectManager::status() const
 {
-    return Compiled;
+    return m_status;
 }
 
 struct RefGuard {
@@ -791,17 +792,85 @@ struct RefGuard {
     IUnknown *p;
 };
 
-bool QSGD3D12GuiThreadShaderEffectManager::reflect(const QByteArray &src, ShaderInfo *result)
+class QSGD3D12ShaderCompileTask : public QRunnable
 {
-    const QString fn = QQmlFile::urlToLocalFileOrQrc(src);
-    QFile f(fn);
-    if (!f.open(QIODevice::ReadOnly)) {
-        qWarning("ShaderEffect: Failed to read %s", qPrintable(fn));
-        return false;
+public:
+    QSGD3D12ShaderCompileTask(QSGD3D12GuiThreadShaderEffectManager *mgr,
+                              QSGD3D12GuiThreadShaderEffectManager::ShaderInfo::Type typeHint,
+                              const QByteArray &src,
+                              QSGD3D12GuiThreadShaderEffectManager::ShaderInfo *result)
+        : mgr(mgr), typeHint(typeHint), src(src), result(result) { }
+
+    void run() override;
+
+private:
+    QSGD3D12GuiThreadShaderEffectManager *mgr;
+    QSGD3D12GuiThreadShaderEffectManager::ShaderInfo::Type typeHint;
+    QByteArray src;
+    QSGD3D12GuiThreadShaderEffectManager::ShaderInfo *result;
+};
+
+void QSGD3D12ShaderCompileTask::run()
+{
+    const char *target = typeHint == QSGD3D12GuiThreadShaderEffectManager::ShaderInfo::TypeVertex ? "vs_5_0" : "ps_5_0";
+    ID3DBlob *bytecode = nullptr;
+    ID3DBlob *errors = nullptr;
+    HRESULT hr = D3DCompile(src.constData(), src.size(), nullptr, nullptr, nullptr,
+                            "main", target, 0, 0, &bytecode, &errors);
+    if (FAILED(hr) || !bytecode) {
+        qWarning("HLSL shader compilation failed: 0x%x", hr);
+        if (errors) {
+            mgr->m_log += QString::fromUtf8(static_cast<const char *>(errors->GetBufferPointer()), errors->GetBufferSize());
+            errors->Release();
+        }
+        mgr->m_status = QSGGuiThreadShaderEffectManager::Error;
+        emit mgr->shaderCodePrepared(false, typeHint, src, result);
+        emit mgr->logAndStatusChanged();
+        return;
     }
-    result->blob = f.readAll();
-    f.close();
 
+    result->blob.resize(bytecode->GetBufferSize());
+    memcpy(result->blob.data(), bytecode->GetBufferPointer(), result->blob.size());
+    bytecode->Release();
+
+    const bool ok = mgr->reflect(result);
+    mgr->m_status = ok ? QSGGuiThreadShaderEffectManager::Compiled : QSGGuiThreadShaderEffectManager::Error;
+    emit mgr->shaderCodePrepared(ok, typeHint, src, result);
+    emit mgr->logAndStatusChanged();
+}
+
+void QSGD3D12GuiThreadShaderEffectManager::prepareShaderCode(ShaderInfo::Type typeHint, const QByteArray &src, ShaderInfo *result)
+{
+    // The D3D12 backend's ShaderEffect implementation supports both HLSL
+    // source strings and bytecode in files as input. The latter is strongly
+    // recommended, but in order to make ShaderEffect users' (and
+    // qtgraphicaleffects') life easier, and since we link to d3dcompiler
+    // anyways, compiling from source is also supported.
+
+    // For simplicity, assume that file = bytecode, string = HLSL.
+    QUrl srcUrl(src);
+    if (!srcUrl.scheme().compare(QLatin1String("qrc"), Qt::CaseInsensitive) || srcUrl.isLocalFile()) {
+        const QString fn = QQmlFile::urlToLocalFileOrQrc(src);
+        QFile f(fn);
+        if (!f.open(QIODevice::ReadOnly)) {
+            qWarning("ShaderEffect: Failed to read %s", qPrintable(fn));
+            emit shaderCodePrepared(false, typeHint, src, result);
+            return;
+        }
+        result->blob = f.readAll();
+        f.close();
+        const bool ok = reflect(result);
+        m_status = ok ? Compiled : Error;
+        emit shaderCodePrepared(ok, typeHint, src, result);
+        emit logAndStatusChanged();
+        return;
+    }
+
+    QThreadPool::globalInstance()->start(new QSGD3D12ShaderCompileTask(this, typeHint, src, result));
+}
+
+bool QSGD3D12GuiThreadShaderEffectManager::reflect(ShaderInfo *result)
+{
     ID3D12ShaderReflection *reflector;
     HRESULT hr = D3DReflect(result->blob.constData(), result->blob.size(), IID_PPV_ARGS(&reflector));
     if (FAILED(hr)) {
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h
index edeaba899b..c36ee1a6e6 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h
@@ -160,7 +160,14 @@ public:
     QString log() const override;
     Status status() const override;
 
-    bool reflect(const QByteArray &src, ShaderInfo *result) override;
+    void prepareShaderCode(ShaderInfo::Type typeHint, const QByteArray &src, ShaderInfo *result) override;
+
+private:
+    bool reflect(ShaderInfo *result);
+    QString m_log;
+    Status m_status = Uncompiled;
+
+    friend class QSGD3D12ShaderCompileTask;
 };
 
 QT_END_NAMESPACE
diff --git a/src/quick/items/qquickgenericshadereffect.cpp b/src/quick/items/qquickgenericshadereffect.cpp
index 47272a2eac..11259a588a 100644
--- a/src/quick/items/qquickgenericshadereffect.cpp
+++ b/src/quick/items/qquickgenericshadereffect.cpp
@@ -61,7 +61,10 @@ QQuickGenericShaderEffect::QQuickGenericShaderEffect(QQuickShaderEffect *item, Q
     , m_mgr(nullptr)
     , m_dirty(0)
 {
+    qRegisterMetaType<QSGGuiThreadShaderEffectManager::ShaderInfo::Type>("ShaderInfo::Type");
     connect(m_item, SIGNAL(windowChanged(QQuickWindow*)), this, SLOT(itemWindowChanged(QQuickWindow*)));
+    for (int i = 0; i < NShader; ++i)
+        m_inProgress[i] = nullptr;
 }
 
 QQuickGenericShaderEffect::~QQuickGenericShaderEffect()
@@ -232,6 +235,10 @@ QSGNode *QQuickGenericShaderEffect::handleUpdatePaintNode(QSGNode *oldNode, QQui
         return nullptr;
     }
 
+    // Do not change anything while a new shader is being reflected or compiled.
+    if (m_inProgress[Vertex] || m_inProgress[Fragment])
+        return node;
+
     // The manager should be already created on the gui thread. Just take that instance.
     QSGGuiThreadShaderEffectManager *mgr = shaderEffectManager();
     if (!mgr) {
@@ -327,6 +334,7 @@ QSGGuiThreadShaderEffectManager *QQuickGenericShaderEffect::shaderEffectManager(
                 connect(m_mgr, SIGNAL(logAndStatusChanged()), m_item, SIGNAL(logChanged()));
                 connect(m_mgr, SIGNAL(logAndStatusChanged()), m_item, SIGNAL(statusChanged()));
                 connect(m_mgr, SIGNAL(textureChanged()), this, SLOT(markGeometryDirtyAndUpdateIfSupportsAtlas()));
+                connect(m_mgr, &QSGGuiThreadShaderEffectManager::shaderCodePrepared, this, &QQuickGenericShaderEffect::shaderCodePrepared);
             }
         } else if (!w) {
             // Wait until itemWindowChanged() gets called. Return null for now.
@@ -377,27 +385,27 @@ void QQuickGenericShaderEffect::disconnectSignals(Shader shaderType)
     }
 }
 
-struct ReflectCache
+struct ShaderInfoCache
 {
     bool contains(const QByteArray &key) const
     {
-        return m_reflectCache.contains(key);
+        return m_shaderInfoCache.contains(key);
     }
 
     QSGGuiThreadShaderEffectManager::ShaderInfo value(const QByteArray &key) const
     {
-        return m_reflectCache.value(key);
+        return m_shaderInfoCache.value(key);
     }
 
     void insert(const QByteArray &key, const QSGGuiThreadShaderEffectManager::ShaderInfo &value)
     {
-        m_reflectCache.insert(key, value);
+        m_shaderInfoCache.insert(key, value);
     }
 
-    QHash<QByteArray, QSGGuiThreadShaderEffectManager::ShaderInfo> m_reflectCache;
+    QHash<QByteArray, QSGGuiThreadShaderEffectManager::ShaderInfo> m_shaderInfoCache;
 };
 
-Q_GLOBAL_STATIC(ReflectCache, reflectCache)
+Q_GLOBAL_STATIC(ShaderInfoCache, shaderInfoCache)
 
 void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray &src)
 {
@@ -413,22 +421,26 @@ void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray
     m_shaders[shaderType].varData.clear();
 
     if (!src.isEmpty()) {
-        // Figure out what input parameters and variables are used in the shader.
-        // For file-based shader source/bytecode this is where the data is pulled
-        // in from the file.
-        if (reflectCache()->contains(src)) {
-            m_shaders[shaderType].shaderInfo = reflectCache()->value(src);
+        if (shaderInfoCache()->contains(src)) {
+            m_shaders[shaderType].shaderInfo = shaderInfoCache()->value(src);
+            m_shaders[shaderType].hasShaderCode = true;
         } else {
-            QSGGuiThreadShaderEffectManager::ShaderInfo shaderInfo;
-            if (!mgr->reflect(src, &shaderInfo)) {
-                qWarning("ShaderEffect: shader reflection failed for %s", src.constData());
-                m_shaders[shaderType].hasShaderCode = false;
-                return;
-            }
-            m_shaders[shaderType].shaderInfo = shaderInfo;
-            reflectCache()->insert(src, shaderInfo);
+            // Each prepareShaderCode call needs its own work area, hence the
+            // dynamic alloc. If there are calls in progress, let those run to
+            // finish, their results can then simply be ignored because
+            // m_inProgress indicates what we care about.
+            m_inProgress[shaderType] = new QSGGuiThreadShaderEffectManager::ShaderInfo;
+            const QSGGuiThreadShaderEffectManager::ShaderInfo::Type typeHint =
+                    shaderType == Vertex ? QSGGuiThreadShaderEffectManager::ShaderInfo::TypeVertex
+                                         : QSGGuiThreadShaderEffectManager::ShaderInfo::TypeFragment;
+            // Figure out what input parameters and variables are used in the
+            // shader. For file-based shader source/bytecode this is where the data
+            // is pulled in from the file. Some backends may choose to do
+            // source->bytecode compilation as well in this step.
+            mgr->prepareShaderCode(typeHint, src, m_inProgress[shaderType]);
+            // the rest is handled in shaderCodePrepared()
+            return;
         }
-        m_shaders[shaderType].hasShaderCode = true;
     } else {
         m_shaders[shaderType].hasShaderCode = false;
         if (shaderType == Fragment) {
@@ -446,6 +458,44 @@ void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray
         }
     }
 
+    updateShaderVars(shaderType);
+}
+
+void QQuickGenericShaderEffect::shaderCodePrepared(bool ok, QSGGuiThreadShaderEffectManager::ShaderInfo::Type typeHint,
+                                                   const QByteArray &src, QSGGuiThreadShaderEffectManager::ShaderInfo *result)
+{
+    const Shader shaderType = typeHint == QSGGuiThreadShaderEffectManager::ShaderInfo::TypeVertex ? Vertex : Fragment;
+
+    // If another call was made to updateShader() for the same shader type in
+    // the meantime then our results are useless, just drop them.
+    if (result != m_inProgress[shaderType]) {
+        delete result;
+        return;
+    }
+
+    m_shaders[shaderType].shaderInfo = *result;
+    delete result;
+    m_inProgress[shaderType] = nullptr;
+
+    if (!ok) {
+        qWarning("ShaderEffect: shader preparation failed for %s\n%s\n", src.constData(), qPrintable(log()));
+        m_shaders[shaderType].hasShaderCode = false;
+        return;
+    }
+
+    m_shaders[shaderType].hasShaderCode = true;
+    shaderInfoCache()->insert(src, m_shaders[shaderType].shaderInfo);
+    updateShaderVars(shaderType);
+}
+
+void QQuickGenericShaderEffect::updateShaderVars(Shader shaderType)
+{
+    QSGGuiThreadShaderEffectManager *mgr = shaderEffectManager();
+    if (!mgr)
+        return;
+
+    const bool texturesSeparate = mgr->hasSeparateSamplerAndTextureObjects();
+
     const int varCount = m_shaders[shaderType].shaderInfo.variables.count();
     m_shaders[shaderType].varData.resize(varCount);
 
diff --git a/src/quick/items/qquickgenericshadereffect_p.h b/src/quick/items/qquickgenericshadereffect_p.h
index ab17a7fb87..5ec83eb60f 100644
--- a/src/quick/items/qquickgenericshadereffect_p.h
+++ b/src/quick/items/qquickgenericshadereffect_p.h
@@ -103,6 +103,8 @@ private slots:
     void markGeometryDirtyAndUpdateIfSupportsAtlas();
     void itemWindowChanged(QQuickWindow *w);
     void backendChanged();
+    void shaderCodePrepared(bool ok, QSGGuiThreadShaderEffectManager::ShaderInfo::Type typeHint,
+                            const QByteArray &src, QSGGuiThreadShaderEffectManager::ShaderInfo *result);
 
 private:
     QSGGuiThreadShaderEffectManager *shaderEffectManager() const;
@@ -113,8 +115,9 @@ private:
 
         NShader
     };
-    void updateShader(Shader which, const QByteArray &src);
-    void disconnectSignals(Shader which);
+    void updateShader(Shader shaderType, const QByteArray &src);
+    void updateShaderVars(Shader shaderType);
+    void disconnectSignals(Shader shaderType);
     bool sourceIsUnique(QQuickItem *source, Shader typeToSkip, int indexToSkip) const;
 
     QQuickShaderEffect *m_item;
@@ -132,6 +135,7 @@ private:
     QSGShaderEffectNode::DirtyShaderFlags m_dirty;
     QSet<int> m_dirtyConstants[NShader];
     QSet<int> m_dirtyTextures[NShader];
+    QSGGuiThreadShaderEffectManager::ShaderInfo *m_inProgress[NShader];
 
     struct SignalMapper {
         SignalMapper() : mapper(nullptr), active(false) { }
diff --git a/src/quick/items/qquickshadereffect.cpp b/src/quick/items/qquickshadereffect.cpp
index f7fc7880ed..4f1a9a28ec 100644
--- a/src/quick/items/qquickshadereffect.cpp
+++ b/src/quick/items/qquickshadereffect.cpp
@@ -209,11 +209,33 @@ QT_BEGIN_NAMESPACE
     it is the textures that map to properties referencing \l Image or
     \l ShaderEffectSource items.
 
-    Unlike with OpenGL, runtime compilation of shader source code may not be
-    supported. Backends for modern APIs are likely to prefer offline
+    Unlike OpenGL, backends for modern APIs will typically prefer offline
     compilation and shipping pre-compiled bytecode with applications instead of
-    inlined shader source strings. To check what is expected at runtime, use the
-    GraphicsInfo.shaderSourceType and GraphicsInfo.shaderCompilationType properties.
+    inlined shader source strings. In this case the string properties for
+    vertex and fragment shaders are treated as URLs referring to local files or
+    files shipped via the Qt resource system.
+
+    To check at runtime what is supported, use the
+    GraphicsInfo.shaderSourceType and GraphicsInfo.shaderCompilationType
+    properties. Note that these are bitmasks, because some backends may support
+    multiple approaches.
+
+    In case of Direct3D 12, both bytecode in files and HLSL source strings are
+    supported. If the vertexShader and fragmentShader properties form a valid
+    URL with the \c file or \c qrc schema, the bytecode is read from the
+    specified file. Otherwise, the string is treated as HLSL source code and is
+    compiled at runtime, assuming Shader Model 5.0 and an entry point of
+    \c{"main"}. This allows dynamically constructing shader strings. However,
+    whenever the shader source code is static, it is strongly recommended to
+    pre-compile to bytecode using the \c fxc tool and refer to these files from
+    QML. This will be a lot more efficient at runtime and allows catching
+    syntax errors in the shaders at compile time.
+
+    Unlike OpenGL, the Direct3D backend is able to perform runtime shader
+    compilation on dedicated threads. This is managed transparently to the
+    applications, and means that ShaderEffect items that contain HLSL source
+    strings do not block the rendering or other parts of the application until
+    the bytecode is ready.
 
     \table 70%
     \row
diff --git a/src/quick/scenegraph/qsgadaptationlayer_p.h b/src/quick/scenegraph/qsgadaptationlayer_p.h
index 8fdcf7af64..179ec3e3fa 100644
--- a/src/quick/scenegraph/qsgadaptationlayer_p.h
+++ b/src/quick/scenegraph/qsgadaptationlayer_p.h
@@ -273,9 +273,10 @@ public:
         uint constantDataSize;
     };
 
-    virtual bool reflect(const QByteArray &src, ShaderInfo *result) = 0;
+    virtual void prepareShaderCode(ShaderInfo::Type typeHint, const QByteArray &src, ShaderInfo *result) = 0;
 
 Q_SIGNALS:
+    void shaderCodePrepared(bool ok, ShaderInfo::Type typeHint, const QByteArray &src, ShaderInfo *result);
     void textureChanged();
     void logAndStatusChanged();
 };
@@ -536,7 +537,8 @@ inline bool QSGDistanceFieldGlyphCache::containsGlyph(glyph_t glyph)
     return glyphData(glyph).texCoord.isValid();
 }
 
-
 QT_END_NAMESPACE
 
+Q_DECLARE_METATYPE(QSGGuiThreadShaderEffectManager::ShaderInfo::Type)
+
 #endif