D3D12: Convert 8-bit glyphs to 32-bit to prevent using R8 formats

Until we figure out why DXGI_FORMAT_R8_UNORM sampling produces weird
results with AMD cards, disable it and convert to RGBA.

The check could be done at runtime based on the device's VendorId
(0x1002), however the shader needs changes too so it would make
things more convoluted. Hence only a compile-time define for now.

Task-number: QTBUG-55330
Change-Id: I73f038cf0aeba742e8ea6669359c8a6e8e400a06
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

5 files changed, 59 insertions, 26 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index 87215d5b54..900dfe7ef7 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -497,14 +497,15 @@ void QSGD3D12Engine::queueTextureResize(uint id, const QSize &size)
     d->queueTextureResize(id, size);
 }
 
-void QSGD3D12Engine::queueTextureUpload(uint id, const QImage &image, const QPoint &dstPos)
+void QSGD3D12Engine::queueTextureUpload(uint id, const QImage &image, const QPoint &dstPos, TextureUploadFlags flags)
 {
-    d->queueTextureUpload(id, QVector<QImage>() << image, QVector<QPoint>() << dstPos);
+    d->queueTextureUpload(id, QVector<QImage>() << image, QVector<QPoint>() << dstPos, flags);
 }
 
-void QSGD3D12Engine::queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos)
+void QSGD3D12Engine::queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos,
+                                        TextureUploadFlags flags)
 {
-    d->queueTextureUpload(id, images, dstPos);
+    d->queueTextureUpload(id, images, dstPos, flags);
 }
 
 void QSGD3D12Engine::releaseTexture(uint id)
@@ -2321,7 +2322,7 @@ uint QSGD3D12EnginePrivate::genTexture()
     return id;
 }
 
-static inline DXGI_FORMAT textureFormat(QImage::Format format, bool wantsAlpha, bool mipmap,
+static inline DXGI_FORMAT textureFormat(QImage::Format format, bool wantsAlpha, bool mipmap, bool force32bit,
                                         QImage::Format *imageFormat, int *bytesPerPixel)
 {
     DXGI_FORMAT f = DXGI_FORMAT_R8G8B8A8_UNORM;
@@ -2333,8 +2334,12 @@ static inline DXGI_FORMAT textureFormat(QImage::Format format, bool wantsAlpha,
         case QImage::Format_Grayscale8:
         case QImage::Format_Indexed8:
         case QImage::Format_Alpha8:
-            f = DXGI_FORMAT_R8_UNORM;
-            bpp = 1;
+            if (!force32bit) {
+                f = DXGI_FORMAT_R8_UNORM;
+                bpp = 1;
+            } else {
+                convFormat = QImage::Format_RGBA8888;
+            }
             break;
         case QImage::Format_RGB32:
             f = DXGI_FORMAT_B8G8R8A8_UNORM;
@@ -2410,7 +2415,9 @@ void QSGD3D12EnginePrivate::createTexture(uint id, const QSize &size, QImage::Fo
     textureDesc.Height = adjustedSize.height();
     textureDesc.DepthOrArraySize = 1;
     textureDesc.MipLevels = !t.mipmap() ? 1 : QSGD3D12Engine::mipMapLevels(adjustedSize);
-    textureDesc.Format = textureFormat(format, t.alpha(), t.mipmap(), nullptr, nullptr);
+    textureDesc.Format = textureFormat(format, t.alpha(), t.mipmap(),
+                                       createFlags.testFlag(QSGD3D12Engine::TextureAlways32Bit),
+                                       nullptr, nullptr);
     textureDesc.SampleDesc.Count = 1;
     textureDesc.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN;
     if (t.mipmap())
@@ -2524,7 +2531,8 @@ void QSGD3D12EnginePrivate::queueTextureResize(uint id, const QSize &size)
         qDebug("submitted old content copy for texture %u on the copy queue, fence %llu", id, t.fenceValue);
 }
 
-void QSGD3D12EnginePrivate::queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos)
+void QSGD3D12EnginePrivate::queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos,
+                                               QSGD3D12Engine::TextureUploadFlags flags)
 {
     Q_ASSERT(id);
     Q_ASSERT(images.count() == dstPos.count());
@@ -2561,7 +2569,9 @@ void QSGD3D12EnginePrivate::queueTextureUpload(uint id, const QVector<QImage> &i
     int totalSize = 0;
     for (const QImage &image : images) {
         int bytesPerPixel;
-        textureFormat(image.format(), t.alpha(), t.mipmap(), nullptr, &bytesPerPixel);
+        textureFormat(image.format(), t.alpha(), t.mipmap(),
+                      flags.testFlag(QSGD3D12Engine::TextureUploadAlways32Bit),
+                      nullptr, &bytesPerPixel);
         const int w = !t.mipmap() ? image.width() : adjustedTextureSize.width();
         const int h = !t.mipmap() ? image.height() : adjustedTextureSize.height();
         const int stride = alignedSize(w * bytesPerPixel, D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
@@ -2593,7 +2603,9 @@ void QSGD3D12EnginePrivate::queueTextureUpload(uint id, const QVector<QImage> &i
     for (int i = 0; i < images.count(); ++i) {
         QImage::Format convFormat;
         int bytesPerPixel;
-        textureFormat(images[i].format(), t.alpha(), t.mipmap(), &convFormat, &bytesPerPixel);
+        textureFormat(images[i].format(), t.alpha(), t.mipmap(),
+                      flags.testFlag(QSGD3D12Engine::TextureUploadAlways32Bit),
+                      &convFormat, &bytesPerPixel);
         if (Q_UNLIKELY(debug_texture() && i == 0))
             qDebug("source image format %d, target format %d, bpp %d", images[i].format(), convFormat, bytesPerPixel);
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
index 46cd73e63a..b30994fe0d 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
@@ -349,16 +349,22 @@ public:
     static QSize mipMapAdjustedSourceSize(const QSize &size);
 
     enum TextureCreateFlag {
-        TextureWithAlpha = 0x1,
-        TextureWithMipMaps = 0x2
+        TextureWithAlpha = 0x01,
+        TextureWithMipMaps = 0x02,
+        TextureAlways32Bit = 0x04
     };
     Q_DECLARE_FLAGS(TextureCreateFlags, TextureCreateFlag)
 
+    enum TextureUploadFlag {
+        TextureUploadAlways32Bit = 0x01
+    };
+    Q_DECLARE_FLAGS(TextureUploadFlags, TextureUploadFlag)
+
     uint genTexture();
     void createTexture(uint id, const QSize &size, QImage::Format format, TextureCreateFlags flags);
     void queueTextureResize(uint id, const QSize &size);
-    void queueTextureUpload(uint id, const QImage &image, const QPoint &dstPos = QPoint());
-    void queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos);
+    void queueTextureUpload(uint id, const QImage &image, const QPoint &dstPos = QPoint(), TextureUploadFlags flags = 0);
+    void queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos, TextureUploadFlags flags = 0);
     void releaseTexture(uint id);
     void useTexture(uint id);
 
@@ -381,6 +387,7 @@ private:
 
 Q_DECLARE_OPERATORS_FOR_FLAGS(QSGD3D12Engine::ClearFlags)
 Q_DECLARE_OPERATORS_FOR_FLAGS(QSGD3D12Engine::TextureCreateFlags)
+Q_DECLARE_OPERATORS_FOR_FLAGS(QSGD3D12Engine::TextureUploadFlags)
 
 QT_END_NAMESPACE
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
index b3b244cd86..1048ed63e7 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
@@ -170,7 +170,8 @@ public:
     uint genTexture();
     void createTexture(uint id, const QSize &size, QImage::Format format, QSGD3D12Engine::TextureCreateFlags flags);
     void queueTextureResize(uint id, const QSize &size);
-    void queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos);
+    void queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos,
+                            QSGD3D12Engine::TextureUploadFlags flags);
     void releaseTexture(uint id);
     void useTexture(uint id);
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
index 45ef202e83..915917c3d5 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
@@ -42,6 +42,11 @@
 
 QT_BEGIN_NAMESPACE
 
+// Convert A8 glyphs to 32-bit in the engine. This is here to work around
+// QTBUG-55330 for AMD cards.
+// If removing, textmask.hlsl must be adjusted! (.a -> .r)
+#define ALWAYS_32BIT
+
 // NOTE: Avoid categorized logging. It is slow.
 
 #define DECLARE_DEBUG_VAR(variable) \
@@ -77,7 +82,11 @@ void QSGD3D12GlyphCache::createTextureData(int width, int height)
 
     const QImage::Format imageFormat =
             m_format == QFontEngine::Format_A8 ? QImage::Format_Alpha8 : QImage::Format_ARGB32_Premultiplied;
-    m_engine->createTexture(m_id, m_size, imageFormat, QSGD3D12Engine::TextureWithAlpha);
+    m_engine->createTexture(m_id, m_size, imageFormat, QSGD3D12Engine::TextureWithAlpha
+#ifdef ALWAYS_32BIT
+                            | QSGD3D12Engine::TextureAlways32Bit
+#endif
+                            );
 }
 
 void QSGD3D12GlyphCache::resizeTextureData(int width, int height)
@@ -146,7 +155,11 @@ void QSGD3D12GlyphCache::endFillTexture()
 
     Q_ASSERT(m_id);
 
-    m_engine->queueTextureUpload(m_id, m_glyphImages, m_glyphPos);
+    m_engine->queueTextureUpload(m_id, m_glyphImages, m_glyphPos
+#ifdef ALWAYS_32BIT
+                                 , QSGD3D12Engine::TextureUploadAlways32Bit
+#endif
+                                 );
 
     // Nothing else left to do, it is up to the text material to call
     // useTexture() which will then add the texture dependency to the frame.
diff --git a/src/plugins/scenegraph/d3d12/shaders/textmask.hlsl b/src/plugins/scenegraph/d3d12/shaders/textmask.hlsl
index f9d92e8ee9..bb9381e7c0 100644
--- a/src/plugins/scenegraph/d3d12/shaders/textmask.hlsl
+++ b/src/plugins/scenegraph/d3d12/shaders/textmask.hlsl
@@ -45,7 +45,7 @@ float4 PS_TextMask32(PSInput input) : SV_TARGET
 
 float4 PS_TextMask8(PSInput input) : SV_TARGET
 {
-    return colorVec * tex.Sample(samp, input.coord).r;
+    return colorVec * tex.Sample(samp, input.coord).a;
 }
 
 struct StyledPSInput
@@ -66,8 +66,8 @@ StyledPSInput VS_StyledText(VSInput input)
 
 float4 PS_StyledText(StyledPSInput input) : SV_TARGET
 {
-    float glyph = tex.Sample(samp, input.coord).r;
-    float style = clamp(tex.Sample(samp, input.shiftedCoord).r - glyph, 0.0, 1.0);
+    float glyph = tex.Sample(samp, input.coord).a;
+    float style = clamp(tex.Sample(samp, input.shiftedCoord).a - glyph, 0.0, 1.0);
     return style * styleColor + glyph * colorVec;
 }
 
@@ -95,10 +95,10 @@ OutlinedPSInput VS_OutlinedText(VSInput input)
 
 float4 PS_OutlinedText(OutlinedPSInput input) : SV_TARGET
 {
-    float glyph = tex.Sample(samp, input.coord).r;
-    float outline = clamp(clamp(tex.Sample(samp, input.coordUp).r
-        + tex.Sample(samp, input.coordDown).r
-        + tex.Sample(samp, input.coordLeft).r
-        + tex.Sample(samp, input.coordRight).r, 0.0, 1.0) - glyph, 0.0, 1.0);
+    float glyph = tex.Sample(samp, input.coord).a;
+    float outline = clamp(clamp(tex.Sample(samp, input.coordUp).a
+        + tex.Sample(samp, input.coordDown).a
+        + tex.Sample(samp, input.coordLeft).a
+        + tex.Sample(samp, input.coordRight).a, 0.0, 1.0) - glyph, 0.0, 1.0);
     return outline * styleColor + glyph * colorVec;
 }