Fix distorted text with subpixel matrix translation

We would pixel-align native text *before* applying the
model-view matrix, which would cause GL_NEAREST artifacts to
show up when the text was positioned at a subpixel offset in
some cases. Instead, we pixel-align the coordinates after mapping
them to the view frustum, but before applying the projection to the
screen.

To make it easier to modify the buffer layout for the shaders the
next time, this also adds some constants for offsets.

[ChangeLog][Text] Fixed an issue where text using NativeRendering
would look slightly skewed if it was inside a parent that had
been positioned at a subpixel offset.

Fixes: QTBUG-96112
Fixes: QTBUG-83626
Task-number: QTBUG-55638
Change-Id: Ifb785ad5830093df94afc75a7bc288e24ca7aa38
Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io>
(cherry picked from commit b21948f5e811ce1b7abf065bc48af61a231e86f4)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

14 files changed, 155 insertions, 40 deletions

diff --git a/src/quick/scenegraph/qsgdefaultglyphnode_p.cpp b/src/quick/scenegraph/qsgdefaultglyphnode_p.cpp
index 95a52a599d..7af1dc9b01 100644
--- a/src/quick/scenegraph/qsgdefaultglyphnode_p.cpp
+++ b/src/quick/scenegraph/qsgdefaultglyphnode_p.cpp
@@ -428,6 +428,18 @@ QSGTextMaskRhiShader::QSGTextMaskRhiShader(QFontEngine::GlyphFormat glyphFormat)
                       QStringLiteral(":/qt-project.org/scenegraph/shaders_ng/textmask.frag.qsb"));
 }
 
+enum UbufOffset {
+    ModelViewMatrixOffset = 0,
+    ProjectionMatrixOffset = ModelViewMatrixOffset + 64,
+    ColorOffset = ProjectionMatrixOffset + 64,
+    TextureScaleOffset = ColorOffset + 16,
+    DprOffset = TextureScaleOffset + 8,
+
+    // + 1 float padding (vec4 must be aligned to 16)
+    StyleColorOffset = DprOffset + 4 + 4,
+    ShiftOffset = StyleColorOffset + 16
+};
+
 bool QSGTextMaskRhiShader::updateUniformData(RenderState &state,
                                              QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
 {
@@ -443,11 +455,14 @@ bool QSGTextMaskRhiShader::updateUniformData(RenderState &state,
 
     bool changed = false;
     QByteArray *buf = state.uniformData();
-    Q_ASSERT(buf->size() >= 92);
+    Q_ASSERT(buf->size() >= DprOffset + 4);
 
     if (state.isMatrixDirty()) {
-        const QMatrix4x4 m = state.combinedMatrix();
-        memcpy(buf->data(), m.constData(), 64);
+        const QMatrix4x4 mv = state.modelViewMatrix();
+        memcpy(buf->data() + ModelViewMatrixOffset, mv.constData(), 64);
+        const QMatrix4x4 p = state.projectionMatrix();
+        memcpy(buf->data() + ProjectionMatrixOffset, p.constData(), 64);
+
         changed = true;
     }
 
@@ -456,13 +471,13 @@ bool QSGTextMaskRhiShader::updateUniformData(RenderState &state,
     if (updated || !oldMat || oldRtex != newRtex) {
         const QVector2D textureScale = QVector2D(1.0f / mat->rhiGlyphCache()->width(),
                                                  1.0f / mat->rhiGlyphCache()->height());
-        memcpy(buf->data() + 64 + 16, &textureScale, 8);
+        memcpy(buf->data() + TextureScaleOffset, &textureScale, 8);
         changed = true;
     }
 
     if (!oldMat) {
         float dpr = state.devicePixelRatio();
-        memcpy(buf->data() + 64 + 16 + 8, &dpr, 4);
+        memcpy(buf->data() + DprOffset, &dpr, 4);
     }
 
     // move texture uploads/copies onto the renderer's soon-to-be-committed list
@@ -510,11 +525,11 @@ bool QSG8BitTextMaskRhiShader::updateUniformData(RenderState &state,
     QSGTextMaskMaterial *oldMat = static_cast<QSGTextMaskMaterial *>(oldMaterial);
 
     QByteArray *buf = state.uniformData();
-    Q_ASSERT(buf->size() >= 80);
+    Q_ASSERT(buf->size() >= ColorOffset + 16);
 
     if (oldMat == nullptr || mat->color() != oldMat->color() || state.isOpacityDirty()) {
         const QVector4D color = qsg_premultiply(mat->color(), state.opacity());
-        memcpy(buf->data() + 64, &color, 16);
+        memcpy(buf->data() + ColorOffset, &color, 16);
         changed = true;
     }
 
@@ -553,12 +568,12 @@ bool QSG24BitTextMaskRhiShader::updateUniformData(RenderState &state,
     QSGTextMaskMaterial *oldMat = static_cast<QSGTextMaskMaterial *>(oldMaterial);
 
     QByteArray *buf = state.uniformData();
-    Q_ASSERT(buf->size() >= 92);
+    Q_ASSERT(buf->size() >= ColorOffset + 16);
 
     if (oldMat == nullptr || mat->color() != oldMat->color() || state.isOpacityDirty()) {
         // shader takes vec4 but uses alpha only; coloring happens via the blend constant
         const QVector4D color = qsg_premultiply(mat->color(), state.opacity());
-        memcpy(buf->data() + 64, &color, 16);
+        memcpy(buf->data() + ColorOffset, &color, 16);
         changed = true;
     }
 
@@ -608,12 +623,12 @@ bool QSG32BitColorTextRhiShader::updateUniformData(RenderState &state,
     QSGTextMaskMaterial *oldMat = static_cast<QSGTextMaskMaterial *>(oldMaterial);
 
     QByteArray *buf = state.uniformData();
-    Q_ASSERT(buf->size() >= 92);
+    Q_ASSERT(buf->size() >= ColorOffset + 16);
 
     if (oldMat == nullptr || mat->color() != oldMat->color() || state.isOpacityDirty()) {
         // shader takes vec4 but uses alpha only
         const QVector4D color(0, 0, 0, mat->color().w() * state.opacity());
-        memcpy(buf->data() + 64, &color, 16);
+        memcpy(buf->data() + ColorOffset, &color, 16);
         changed = true;
     }
 
@@ -649,20 +664,17 @@ bool QSGStyledTextRhiShader::updateUniformData(RenderState &state,
     QSGStyledTextMaterial *oldMat = static_cast<QSGStyledTextMaterial *>(oldMaterial);
 
     QByteArray *buf = state.uniformData();
-    Q_ASSERT(buf->size() >= 120);
-
-    // matrix..dpr + 1 float padding (vec4 must be aligned to 16)
-    const int startOffset = 64 + 16 + 8 + 4 + 4;
+    Q_ASSERT(buf->size() >= ShiftOffset + 8);
 
     if (oldMat == nullptr || mat->styleColor() != oldMat->styleColor() || state.isOpacityDirty()) {
         const QVector4D styleColor = qsg_premultiply(mat->styleColor(), state.opacity());
-        memcpy(buf->data() + startOffset, &styleColor, 16);
+        memcpy(buf->data() + StyleColorOffset, &styleColor, 16);
         changed = true;
     }
 
     if (oldMat == nullptr || oldMat->styleShift() != mat->styleShift()) {
         const QVector2D v = mat->styleShift();
-        memcpy(buf->data() + startOffset + 16, &v, 8);
+        memcpy(buf->data() + ShiftOffset, &v, 8);
         changed = true;
     }
 
diff --git a/src/quick/scenegraph/shaders_ng/24bittextmask.frag b/src/quick/scenegraph/shaders_ng/24bittextmask.frag
index bc3826a924..ed8da4cd30 100644
--- a/src/quick/scenegraph/shaders_ng/24bittextmask.frag
+++ b/src/quick/scenegraph/shaders_ng/24bittextmask.frag
@@ -6,8 +6,9 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
-    vec4 color; // only alpha is used, but must be vec4 due to layout compat
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
+    vec4 color;
     vec2 textureScale;
     float dpr;
 } ubuf;
diff --git a/src/quick/scenegraph/shaders_ng/32bitcolortext.frag b/src/quick/scenegraph/shaders_ng/32bitcolortext.frag
index 63e445f90b..4198a4d339 100644
--- a/src/quick/scenegraph/shaders_ng/32bitcolortext.frag
+++ b/src/quick/scenegraph/shaders_ng/32bitcolortext.frag
@@ -6,8 +6,9 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
-    vec4 color; // only alpha is used, but must be vec4 due to layout compat
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
+    vec4 color;
     vec2 textureScale;
     float dpr;
 } ubuf;
diff --git a/src/quick/scenegraph/shaders_ng/8bittextmask.frag b/src/quick/scenegraph/shaders_ng/8bittextmask.frag
index 6304e821ff..a06743876d 100644
--- a/src/quick/scenegraph/shaders_ng/8bittextmask.frag
+++ b/src/quick/scenegraph/shaders_ng/8bittextmask.frag
@@ -6,7 +6,8 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
diff --git a/src/quick/scenegraph/shaders_ng/8bittextmask_a.frag b/src/quick/scenegraph/shaders_ng/8bittextmask_a.frag
index 0d0fa1cd3a..f725cbc5e7 100644
--- a/src/quick/scenegraph/shaders_ng/8bittextmask_a.frag
+++ b/src/quick/scenegraph/shaders_ng/8bittextmask_a.frag
@@ -6,7 +6,8 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
diff --git a/src/quick/scenegraph/shaders_ng/outlinedtext.frag b/src/quick/scenegraph/shaders_ng/outlinedtext.frag
index 947d161a50..e2f82d3845 100644
--- a/src/quick/scenegraph/shaders_ng/outlinedtext.frag
+++ b/src/quick/scenegraph/shaders_ng/outlinedtext.frag
@@ -11,11 +11,12 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    // must match styledtext
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
+    // the above must stay compatible with textmask/8bittextmask
     vec4 styleColor;
     vec2 shift;
 } ubuf;
diff --git a/src/quick/scenegraph/shaders_ng/outlinedtext.vert b/src/quick/scenegraph/shaders_ng/outlinedtext.vert
index 023f9dfdc2..4068e42f28 100644
--- a/src/quick/scenegraph/shaders_ng/outlinedtext.vert
+++ b/src/quick/scenegraph/shaders_ng/outlinedtext.vert
@@ -10,11 +10,12 @@ layout(location = 3) out vec2 sCoordLeft;
 layout(location = 4) out vec2 sCoordRight;
 
 layout(std140, binding = 0) uniform buf {
-    // must match styledtext
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
+    // the above must stay compatible with textmask/8bittextmask
     vec4 styleColor;
     vec2 shift;
 } ubuf;
@@ -28,6 +29,6 @@ void main()
      sCoordDown = (tCoord - vec2(0.0, 1.0)) * ubuf.textureScale;
      sCoordLeft = (tCoord - vec2(-1.0, 0.0)) * ubuf.textureScale;
      sCoordRight = (tCoord - vec2(1.0, 0.0)) * ubuf.textureScale;
-     vec3 dprSnapPos = floor(vCoord.xyz * ubuf.dpr + 0.5) / ubuf.dpr;
-     gl_Position = ubuf.matrix * vec4(dprSnapPos, vCoord.w);
+     vec4 xformed = ubuf.modelViewMatrix * vCoord;
+     gl_Position = ubuf.projectionMatrix * vec4(floor(xformed.xyz * ubuf.dpr + 0.5) / ubuf.dpr, xformed.w);
 }
diff --git a/src/quick/scenegraph/shaders_ng/outlinedtext_a.frag b/src/quick/scenegraph/shaders_ng/outlinedtext_a.frag
index 5b7bd9ca82..274d891a3c 100644
--- a/src/quick/scenegraph/shaders_ng/outlinedtext_a.frag
+++ b/src/quick/scenegraph/shaders_ng/outlinedtext_a.frag
@@ -11,11 +11,12 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    // must match styledtext
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
+    // the above must stay compatible with textmask/8bittextmask
     vec4 styleColor;
     vec2 shift;
 } ubuf;
diff --git a/src/quick/scenegraph/shaders_ng/styledtext.frag b/src/quick/scenegraph/shaders_ng/styledtext.frag
index 0b16396037..2e380dfeae 100644
--- a/src/quick/scenegraph/shaders_ng/styledtext.frag
+++ b/src/quick/scenegraph/shaders_ng/styledtext.frag
@@ -8,7 +8,8 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
diff --git a/src/quick/scenegraph/shaders_ng/styledtext.vert b/src/quick/scenegraph/shaders_ng/styledtext.vert
index beadf07c79..271dae8d8a 100644
--- a/src/quick/scenegraph/shaders_ng/styledtext.vert
+++ b/src/quick/scenegraph/shaders_ng/styledtext.vert
@@ -7,7 +7,8 @@ layout(location = 0) out vec2 sampleCoord;
 layout(location = 1) out vec2 shiftedSampleCoord;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
@@ -22,6 +23,6 @@ void main()
 {
      sampleCoord = tCoord * ubuf.textureScale;
      shiftedSampleCoord = (tCoord - ubuf.shift) * ubuf.textureScale;
-     vec3 dprSnapPos = floor(vCoord.xyz * ubuf.dpr + 0.5) / ubuf.dpr;
-     gl_Position = ubuf.matrix * vec4(dprSnapPos, vCoord.w);
+     vec4 xformed = ubuf.modelViewMatrix * vCoord;
+     gl_Position = ubuf.projectionMatrix * vec4(floor(xformed.xyz * ubuf.dpr + 0.5) / ubuf.dpr, xformed.w);
 }
diff --git a/src/quick/scenegraph/shaders_ng/styledtext_a.frag b/src/quick/scenegraph/shaders_ng/styledtext_a.frag
index b673137895..62e162c851 100644
--- a/src/quick/scenegraph/shaders_ng/styledtext_a.frag
+++ b/src/quick/scenegraph/shaders_ng/styledtext_a.frag
@@ -8,7 +8,8 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
diff --git a/src/quick/scenegraph/shaders_ng/textmask.frag b/src/quick/scenegraph/shaders_ng/textmask.frag
index 518d5c965f..ed8da4cd30 100644
--- a/src/quick/scenegraph/shaders_ng/textmask.frag
+++ b/src/quick/scenegraph/shaders_ng/textmask.frag
@@ -6,7 +6,8 @@ layout(location = 0) out vec4 fragColor;
 layout(binding = 1) uniform sampler2D _qt_texture;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
diff --git a/src/quick/scenegraph/shaders_ng/textmask.vert b/src/quick/scenegraph/shaders_ng/textmask.vert
index 9d80d5dadb..e0b3c01bce 100644
--- a/src/quick/scenegraph/shaders_ng/textmask.vert
+++ b/src/quick/scenegraph/shaders_ng/textmask.vert
@@ -6,7 +6,8 @@ layout(location = 1) in vec2 tCoord;
 layout(location = 0) out vec2 sampleCoord;
 
 layout(std140, binding = 0) uniform buf {
-    mat4 matrix;
+    mat4 modelViewMatrix;
+    mat4 projectionMatrix;
     vec4 color;
     vec2 textureScale;
     float dpr;
@@ -17,6 +18,6 @@ out gl_PerVertex { vec4 gl_Position; };
 void main()
 {
      sampleCoord = tCoord * ubuf.textureScale;
-     vec3 dprSnapPos = floor(vCoord.xyz * ubuf.dpr + 0.5) / ubuf.dpr;
-     gl_Position = ubuf.matrix * vec4(dprSnapPos, vCoord.w);
+     vec4 xformed = ubuf.modelViewMatrix * vCoord;
+     gl_Position = ubuf.projectionMatrix * vec4(floor(xformed.xyz * ubuf.dpr + 0.5) / ubuf.dpr, xformed.w);
 }
diff --git a/tests/manual/scenegraph_lancelot/data/text/text_nativerendering_subpixelpositions.qml b/tests/manual/scenegraph_lancelot/data/text/text_nativerendering_subpixelpositions.qml
new file mode 100644
index 0000000000..c60fc4d8b0
--- /dev/null
+++ b/tests/manual/scenegraph_lancelot/data/text/text_nativerendering_subpixelpositions.qml
@@ -0,0 +1,91 @@
+import QtQuick 2.0
+
+//vary font style, native rendering at non-integer offsets
+
+Item {
+    id: topLevel
+    width: 320
+    height: 580
+
+    Repeater {
+        model: [Text.Normal, Text.Outline, Text.Raised, Text.Sunken]
+        Text {
+            y: 20 * index
+            clip: true
+            renderType: Text.NativeRendering
+            width: parent.width
+            wrapMode: Text.Wrap
+            font.pointSize: 10
+            style: modelData
+            styleColor: "green"
+            text: "The quick fox jumps in style " + modelData
+        }
+    }
+
+    Repeater {
+        model: [Text.Normal, Text.Outline, Text.Raised, Text.Sunken]
+        Text {
+            y: 100.5 + 20 * index
+            clip: true
+            renderType: Text.NativeRendering
+            width: parent.width
+            wrapMode: Text.Wrap
+            font.pointSize: 10
+            style: modelData
+            styleColor: "green"
+            text: "The quick fox jumps in style " + modelData
+        }
+    }
+
+    Repeater {
+        model: [Text.Normal, Text.Outline, Text.Raised, Text.Sunken]
+        Text {
+            y: 200.5 + 20 * index
+            x: 0.5
+            clip: true
+            renderType: Text.NativeRendering
+            width: parent.width
+            wrapMode: Text.Wrap
+            font.pointSize: 10
+            style: modelData
+            styleColor: "green"
+            text: "The quick fox jumps in style " + modelData
+        }
+    }
+
+    Repeater {
+        model: [Text.Normal, Text.Outline, Text.Raised, Text.Sunken]
+        Text {
+            y: 300.5 + 20 * index
+            x: 0.5
+            clip: true
+            renderType: Text.NativeRendering
+            width: parent.width
+            wrapMode: Text.Wrap
+            font.pointSize: 10
+            style: modelData
+            styleColor: "green"
+            text: "The quick fox jumps in style " + modelData
+        }
+    }
+
+    Repeater {
+        model: [Text.Normal, Text.Outline, Text.Raised, Text.Sunken]
+        Rectangle {
+            y: 400.5 + 20 * index
+            x: 0.5
+            width: topLevel.width
+            height: topLevel.height
+            clip: true
+            Text {
+                renderType: Text.NativeRendering
+                width: parent.width
+                wrapMode: Text.Wrap
+                font.pointSize: 10
+                style: modelData
+                styleColor: "green"
+                text: "The quick fox jumps in style " + modelData
+            }
+        }
+    }
+}