D3D12: Handle special shader effect cases

Like when there is only one shader is given by the ShaderEffect item, or
when texture providers disappear (use a dummy texture in this case).
Changing the cull mode works now too. Fix also the inconsistent naming
(activate -> use) for textures in the glpyh cache. The cbuffer is now
correctly aligned. Filtering and wrap modes are now taken from the
texture, meaning 'smooth' and 'wrapMode' works.

Change-Id: I265a7676d87d470dc8b3a8f59ea64bdadbc58c5d
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

7 files changed, 95 insertions, 41 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12builtinmaterials.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12builtinmaterials.cpp
index 37a3ab2138..76ecfb9d59 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12builtinmaterials.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12builtinmaterials.cpp
@@ -566,7 +566,7 @@ QSGD3D12Material::UpdateResults QSGD3D12TextMaterial::updatePipeline(const Rende
     tv.addressModeHoriz = QSGD3D12TextureView::AddressClamp;
     tv.addressModeVert = QSGD3D12TextureView::AddressClamp;
 
-    glyphCache()->activateTexture();
+    glyphCache()->useTexture();
 
     return r;
 }
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
index 2fc2701121..45ef202e83 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
@@ -149,7 +149,7 @@ void QSGD3D12GlyphCache::endFillTexture()
     m_engine->queueTextureUpload(m_id, m_glyphImages, m_glyphPos);
 
     // Nothing else left to do, it is up to the text material to call
-    // activateTexture() which will then add the texture dependency to the frame.
+    // useTexture() which will then add the texture dependency to the frame.
 
     m_glyphImages.clear();
     m_glyphPos.clear();
@@ -170,7 +170,7 @@ int QSGD3D12GlyphCache::maxTextureHeight() const
     return 16384;
 }
 
-void QSGD3D12GlyphCache::activateTexture()
+void QSGD3D12GlyphCache::useTexture()
 {
     if (m_id)
         m_engine->useTexture(m_id);
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache_p.h
index a7430580a8..88d3d36f33 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache_p.h
@@ -72,7 +72,7 @@ public:
     int maxTextureWidth() const override;
     int maxTextureHeight() const override;
 
-    void activateTexture();
+    void useTexture();
     QSize currentSize() const;
 
 private:
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp
index a94cdf2827..36f5cf26be 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode.cpp
@@ -39,6 +39,7 @@
 
 #include "qsgd3d12shadereffectnode_p.h"
 #include "qsgd3d12rendercontext_p.h"
+#include "qsgd3d12texture_p.h"
 #include "qsgd3d12engine_p.h"
 #include <QtCore/qfile.h>
 #include <QtQml/qqmlfile.h>
@@ -181,6 +182,11 @@ QSGD3D12ShaderEffectMaterial::QSGD3D12ShaderEffectMaterial(QSGD3D12ShaderEffectN
     setFlag(Blending | RequiresFullMatrix, true); // may be changed in sync()
 }
 
+QSGD3D12ShaderEffectMaterial::~QSGD3D12ShaderEffectMaterial()
+{
+    delete dummy;
+}
+
 struct QSGD3D12ShaderMaterialTypeCache
 {
     QSGMaterialType *get(const QByteArray &vs, const QByteArray &fs);
@@ -260,7 +266,7 @@ int QSGD3D12ShaderEffectMaterial::compare(const QSGMaterial *other) const
 
 int QSGD3D12ShaderEffectMaterial::constantBufferSize() const
 {
-    return linker.constantBufferSize;
+    return QSGD3D12Engine::alignedConstantBufferSize(linker.constantBufferSize);
 }
 
 void QSGD3D12ShaderEffectMaterial::preparePipeline(QSGD3D12PipelineState *pipelineState)
@@ -279,7 +285,7 @@ static inline QColor qsg_premultiply_color(const QColor &c)
 }
 
 QSGD3D12Material::UpdateResults QSGD3D12ShaderEffectMaterial::updatePipeline(const RenderState &state,
-                                                                             QSGD3D12PipelineState *,
+                                                                             QSGD3D12PipelineState *pipelineState,
                                                                              ExtraState *,
                                                                              quint8 *constantBuffer)
 {
@@ -413,15 +419,44 @@ QSGD3D12Material::UpdateResults QSGD3D12ShaderEffectMaterial::updatePipeline(con
         }
     }
 
-    for (QSGTextureProvider *tp : textureProviders) {
+    for (int i = 0; i < textureProviders.count(); ++i) {
+        QSGTextureProvider *tp = textureProviders[i];
+        QSGD3D12TextureView &tv(pipelineState->shaders.rootSig.textureViews[i]);
         if (tp) {
-            if (QSGTexture *t = tp->texture())
+            if (QSGTexture *t = tp->texture()) {
+                tv.filter = t->filtering() == QSGTexture::Linear
+                        ? QSGD3D12TextureView::FilterLinear : QSGD3D12TextureView::FilterNearest;
+                tv.addressModeHoriz = t->horizontalWrapMode() == QSGTexture::ClampToEdge
+                        ? QSGD3D12TextureView::AddressClamp : QSGD3D12TextureView::AddressWrap;
+                tv.addressModeVert = t->verticalWrapMode() == QSGTexture::ClampToEdge
+                        ? QSGD3D12TextureView::AddressClamp : QSGD3D12TextureView::AddressWrap;
                 t->bind();
+                continue;
+            }
+        }
+        if (!dummy) {
+            dummy = new QSGD3D12Texture(node->renderContext()->engine());
+            QImage img(128, 128, QImage::Format_ARGB32_Premultiplied);
+            img.fill(0);
+            dummy->setImage(img, QSGRenderContext::CreateTexture_Alpha);
         }
-        // ### have a dummy in case the texture provider is null
+        tv.filter = QSGD3D12TextureView::FilterNearest;
+        tv.addressModeHoriz = QSGD3D12TextureView::AddressWrap;
+        tv.addressModeVert = QSGD3D12TextureView::AddressWrap;
+        dummy->bind();
     }
 
-    // ### cull mode
+    switch (cullMode) {
+    case QSGShaderEffectNode::FrontFaceCulling:
+        pipelineState->cullMode = QSGD3D12PipelineState::CullFront;
+        break;
+    case QSGShaderEffectNode::BackFaceCulling:
+        pipelineState->cullMode = QSGD3D12PipelineState::CullBack;
+        break;
+    default:
+        pipelineState->cullMode = QSGD3D12PipelineState::CullNone;
+        break;
+    }
 
     return r;
 }
@@ -506,7 +541,7 @@ void QSGD3D12ShaderEffectNode::sync(SyncData *syncData)
     if (syncData->dirty & QSGShaderEffectNode::DirtyShaders) {
         QByteArray vertBlob, fragBlob;
 
-        m_material.hasCustomVertexShader = syncData->vertex.shader->valid;
+        m_material.hasCustomVertexShader = syncData->vertex.shader->hasShaderCode;
         if (m_material.hasCustomVertexShader) {
             vertBlob = syncData->vertex.shader->shaderInfo.blob;
         } else {
@@ -514,7 +549,7 @@ void QSGD3D12ShaderEffectNode::sync(SyncData *syncData)
                                                sizeof(g_VS_DefaultShaderEffect));
         }
 
-        m_material.hasCustomFragmentShader = syncData->fragment.shader->valid;
+        m_material.hasCustomFragmentShader = syncData->fragment.shader->hasShaderCode;
         if (m_material.hasCustomFragmentShader) {
             fragBlob = syncData->fragment.shader->shaderInfo.blob;
         } else {
@@ -528,11 +563,8 @@ void QSGD3D12ShaderEffectNode::sync(SyncData *syncData)
         if (m_material.hasCustomVertexShader) {
             m_material.linker.feedVertexInput(*syncData->vertex.shader);
             m_material.linker.feedConstants(*syncData->vertex.shader);
-            m_material.linker.feedSamplers(*syncData->vertex.shader);
-            m_material.linker.feedTextures(*syncData->vertex.shader);
         } else {
             QSGShaderEffectNode::ShaderData defaultSD;
-            defaultSD.valid = true;
             defaultSD.shaderInfo.blob = vertBlob;
             defaultSD.shaderInfo.type = QSGGuiThreadShaderEffectManager::ShaderInfo::TypeVertex;
 
@@ -558,13 +590,13 @@ void QSGD3D12ShaderEffectNode::sync(SyncData *syncData)
             m_material.linker.feedConstants(defaultSD);
         }
 
+        m_material.linker.feedSamplers(*syncData->vertex.shader);
+        m_material.linker.feedTextures(*syncData->vertex.shader);
+
         if (m_material.hasCustomFragmentShader) {
             m_material.linker.feedConstants(*syncData->fragment.shader);
-            m_material.linker.feedSamplers(*syncData->fragment.shader);
-            m_material.linker.feedTextures(*syncData->fragment.shader);
         } else {
             QSGShaderEffectNode::ShaderData defaultSD;
-            defaultSD.valid = true;
             defaultSD.shaderInfo.blob = fragBlob;
             defaultSD.shaderInfo.type = QSGGuiThreadShaderEffectManager::ShaderInfo::TypeFragment;
 
@@ -599,6 +631,11 @@ void QSGD3D12ShaderEffectNode::sync(SyncData *syncData)
             m_material.linker.feedTextures(defaultSD);
         }
 
+        // While this may seem unnecessary for the built-in shaders, the value
+        // of 'source' is still in there and we have to process it.
+        m_material.linker.feedSamplers(*syncData->fragment.shader);
+        m_material.linker.feedTextures(*syncData->fragment.shader);
+
         m_material.updateTextureProviders(true);
         markDirty(QSGNode::DirtyMaterial);
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h
index d58233481a..d3e51e5dce 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12shadereffectnode_p.h
@@ -59,6 +59,7 @@ QT_BEGIN_NAMESPACE
 class QSGD3D12RenderContext;
 class QSGD3D12GuiThreadShaderEffectManager;
 class QSGD3D12ShaderEffectNode;
+class QSGD3D12Texture;
 
 class QSGD3D12ShaderLinker
 {
@@ -97,6 +98,7 @@ class QSGD3D12ShaderEffectMaterial : public QSGD3D12Material
 {
 public:
     QSGD3D12ShaderEffectMaterial(QSGD3D12ShaderEffectNode *node);
+    ~QSGD3D12ShaderEffectMaterial();
 
     QSGMaterialType *type() const override;
     int compare(const QSGMaterial *other) const override;
@@ -118,6 +120,7 @@ public:
     QSGD3D12ShaderLinker linker;
     QSGMaterialType *mtype = nullptr;
     QVector<QSGTextureProvider *> textureProviders;
+    QSGD3D12Texture *dummy = nullptr;
 };
 
 class QSGD3D12ShaderEffectNode : public QObject, public QSGShaderEffectNode
@@ -134,6 +137,8 @@ public:
 
     void preprocess() override;
 
+    QSGD3D12RenderContext *renderContext() { return m_rc; }
+
 private Q_SLOTS:
     void handleTextureChange();
     void handleTextureProviderDestroyed(QObject *object);
diff --git a/src/quick/items/qquickgenericshadereffect.cpp b/src/quick/items/qquickgenericshadereffect.cpp
index 0e88868aea..ae34a65a2f 100644
--- a/src/quick/items/qquickgenericshadereffect.cpp
+++ b/src/quick/items/qquickgenericshadereffect.cpp
@@ -389,30 +389,44 @@ void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray
     if (!mgr)
         return;
 
+    const bool texturesSeparate = mgr->hasSeparateSamplerAndTextureObjects();
+
     disconnectSignals(shaderType);
 
+    m_shaders[shaderType].shaderInfo = QSGGuiThreadShaderEffectManager::ShaderInfo();
     m_shaders[shaderType].varData.clear();
 
-    if (src.isEmpty()) {
-        m_shaders[shaderType].valid = false;
-        return;
-    }
-
-    // Figure out what input parameters and variables are used in the shader.
-    // For file-based shader source/bytecode this is where the data is pulled
-    // in from the file.
-    QSGGuiThreadShaderEffectManager::ShaderInfo shaderInfo;
-    // ### this will need some sort of caching mechanism
-    if (!mgr->reflect(src, &shaderInfo)) {
-        qWarning("ShaderEffect: shader reflection failed for %s", src.constData());
-        m_shaders[shaderType].valid = false;
-        return;
+    if (!src.isEmpty()) {
+        // Figure out what input parameters and variables are used in the shader.
+        // For file-based shader source/bytecode this is where the data is pulled
+        // in from the file.
+        QSGGuiThreadShaderEffectManager::ShaderInfo shaderInfo;
+        // ### this may need some sort of caching mechanism
+        if (!mgr->reflect(src, &shaderInfo)) {
+            qWarning("ShaderEffect: shader reflection failed for %s", src.constData());
+            m_shaders[shaderType].hasShaderCode = false;
+            return;
+        }
+        m_shaders[shaderType].shaderInfo = shaderInfo;
+        m_shaders[shaderType].hasShaderCode = true;
+    } else {
+        m_shaders[shaderType].hasShaderCode = false;
+        if (shaderType == Fragment) {
+            // With built-in shaders hasShaderCode is set to false and all
+            // metadata is empty, as it is left up to the node to provide a
+            // built-in default shader and its metadata. However, in case of
+            // the built-in fragment shader the value for 'source' has to be
+            // provided and monitored like with an application-provided shader.
+            QSGGuiThreadShaderEffectManager::ShaderInfo::Variable v;
+            v.name = QByteArrayLiteral("source");
+            v.bindPoint = 0;
+            v.type = texturesSeparate ? QSGGuiThreadShaderEffectManager::ShaderInfo::Texture
+                                      : QSGGuiThreadShaderEffectManager::ShaderInfo::Sampler;
+            m_shaders[shaderType].shaderInfo.variables.append(v);
+        }
     }
 
-    m_shaders[shaderType].shaderInfo = shaderInfo;
-    m_shaders[shaderType].valid = true;
-
-    const int varCount = shaderInfo.variables.count();
+    const int varCount = m_shaders[shaderType].shaderInfo.variables.count();
     m_shaders[shaderType].varData.resize(varCount);
 
     // Reuse signal mappers as much as possible since the mapping is based on
@@ -420,12 +434,10 @@ void QQuickGenericShaderEffect::updateShader(Shader shaderType, const QByteArray
     if (m_signalMappers[shaderType].count() < varCount)
         m_signalMappers[shaderType].resize(varCount);
 
-    const bool texturesSeparate = mgr->hasSeparateSamplerAndTextureObjects();
-
     // Hook up the signals to get notified about changes for properties that
     // correspond to variables in the shader. Store also the values.
     for (int i = 0; i < varCount; ++i) {
-        const auto &v(shaderInfo.variables.at(i));
+        const auto &v(m_shaders[shaderType].shaderInfo.variables.at(i));
         QSGShaderEffectNode::VariableData &vd(m_shaders[shaderType].varData[i]);
         const bool isSpecial = v.name.startsWith("qt_"); // special names not mapped to properties
         if (isSpecial) {
diff --git a/src/quick/scenegraph/qsgadaptationlayer_p.h b/src/quick/scenegraph/qsgadaptationlayer_p.h
index c06e681d30..09690acf56 100644
--- a/src/quick/scenegraph/qsgadaptationlayer_p.h
+++ b/src/quick/scenegraph/qsgadaptationlayer_p.h
@@ -332,8 +332,8 @@ public:
     };
 
     struct ShaderData {
-        ShaderData() : valid(false) { }
-        bool valid;
+        ShaderData() : hasShaderCode(false) { }
+        bool hasShaderCode;
         QSGGuiThreadShaderEffectManager::ShaderInfo shaderInfo;
         QVector<VariableData> varData;
     };