Freetype: Fix device pixel ratio with large fonts

When the dpr scale caused pixel sizes to reach 64 pixels and up,
thus triggering the outline drawing fallback, we would render
the glyphs at the original size but scale the glyph positions,
giving broken rendering. This was because we did not actually
pass the matrix to the fallback function when getting the alpha
maps.

Fixing it revealed a different problem, which was that the bounding
box for the glyphs, returned by alphaMapBoundingBox() did not
actually apply the transform when outline drawing was enabled.

[ChangeLog][QtGui][Text] Fixed rendering of large fonts when a
device pixel ratio is set and the Freetype engine is used.

Task-number: QTBUG-55856
Change-Id: I03d9066faf0e4346628a4eb630f0dd74a81ef148
Reviewed-by: Konstantin Ritt <ritt.ks@gmail.com>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

1 files changed, 52 insertions, 39 deletions

diff --git a/src/gui/text/qfontengine_ft.cpp b/src/gui/text/qfontengine_ft.cpp
index 6d575e0e88..c7b932d4ab 100644
--- a/src/gui/text/qfontengine_ft.cpp
+++ b/src/gui/text/qfontengine_ft.cpp
@@ -889,6 +889,42 @@ static inline bool areMetricsTooLarge(const QFontEngineFT::GlyphInfo &info)
             || (uchar)(info.height) != info.height;
 }
 
+static inline void transformBoundingBox(int *left, int *top, int *right, int *bottom, FT_Matrix *matrix)
+{
+    int l, r, t, b;
+    FT_Vector vector;
+    vector.x = *left;
+    vector.y = *top;
+    FT_Vector_Transform(&vector, matrix);
+    l = r = vector.x;
+    t = b = vector.y;
+    vector.x = *right;
+    vector.y = *top;
+    FT_Vector_Transform(&vector, matrix);
+    if (l > vector.x) l = vector.x;
+    if (r < vector.x) r = vector.x;
+    if (t < vector.y) t = vector.y;
+    if (b > vector.y) b = vector.y;
+    vector.x = *right;
+    vector.y = *bottom;
+    FT_Vector_Transform(&vector, matrix);
+    if (l > vector.x) l = vector.x;
+    if (r < vector.x) r = vector.x;
+    if (t < vector.y) t = vector.y;
+    if (b > vector.y) b = vector.y;
+    vector.x = *left;
+    vector.y = *bottom;
+    FT_Vector_Transform(&vector, matrix);
+    if (l > vector.x) l = vector.x;
+    if (r < vector.x) r = vector.x;
+    if (t < vector.y) t = vector.y;
+    if (b > vector.y) b = vector.y;
+    *left = l;
+    *right = r;
+    *top = t;
+    *bottom = b;
+}
+
 QFontEngineFT::Glyph *QFontEngineFT::loadGlyph(QGlyphSet *set, uint glyph,
                                                QFixed subPixelPosition,
                                                GlyphFormat format,
@@ -978,10 +1014,19 @@ QFontEngineFT::Glyph *QFontEngineFT::loadGlyph(QGlyphSet *set, uint glyph,
     info.yOff = 0;
 
     if ((set && set->outline_drawing) || fetchMetricsOnly) {
-        int left  = FLOOR(slot->metrics.horiBearingX);
-        int right = CEIL(slot->metrics.horiBearingX + slot->metrics.width);
-        int top    = CEIL(slot->metrics.horiBearingY);
-        int bottom = FLOOR(slot->metrics.horiBearingY - slot->metrics.height);
+        int left  = slot->metrics.horiBearingX;
+        int right = slot->metrics.horiBearingX + slot->metrics.width;
+        int top    = slot->metrics.horiBearingY;
+        int bottom = slot->metrics.horiBearingY - slot->metrics.height;
+
+        if (transform && slot->format != FT_GLYPH_FORMAT_BITMAP)
+            transformBoundingBox(&left, &top, &right, &bottom, &matrix);
+
+        left = FLOOR(left);
+        right = CEIL(right);
+        bottom = FLOOR(bottom);
+        top = CEIL(top);
+
         info.x = TRUNC(left);
         info.y = TRUNC(top);
         info.width = TRUNC(right - left);
@@ -1044,40 +1089,8 @@ QFontEngineFT::Glyph *QFontEngineFT::loadGlyph(QGlyphSet *set, uint glyph,
     int right = slot->metrics.horiBearingX + slot->metrics.width;
     int top    = slot->metrics.horiBearingY;
     int bottom = slot->metrics.horiBearingY - slot->metrics.height;
-    if(transform && slot->format != FT_GLYPH_FORMAT_BITMAP) {
-        int l, r, t, b;
-        FT_Vector vector;
-        vector.x = left;
-        vector.y = top;
-        FT_Vector_Transform(&vector, &matrix);
-        l = r = vector.x;
-        t = b = vector.y;
-        vector.x = right;
-        vector.y = top;
-        FT_Vector_Transform(&vector, &matrix);
-        if (l > vector.x) l = vector.x;
-        if (r < vector.x) r = vector.x;
-        if (t < vector.y) t = vector.y;
-        if (b > vector.y) b = vector.y;
-        vector.x = right;
-        vector.y = bottom;
-        FT_Vector_Transform(&vector, &matrix);
-        if (l > vector.x) l = vector.x;
-        if (r < vector.x) r = vector.x;
-        if (t < vector.y) t = vector.y;
-        if (b > vector.y) b = vector.y;
-        vector.x = left;
-        vector.y = bottom;
-        FT_Vector_Transform(&vector, &matrix);
-        if (l > vector.x) l = vector.x;
-        if (r < vector.x) r = vector.x;
-        if (t < vector.y) t = vector.y;
-        if (b > vector.y) b = vector.y;
-        left = l;
-        right = r;
-        top = t;
-        bottom = b;
-    }
+    if (transform && slot->format != FT_GLYPH_FORMAT_BITMAP)
+        transformBoundingBox(&left, &top, &right, &bottom, &matrix);
     left = FLOOR(left);
     right = CEIL(right);
     bottom = FLOOR(bottom);
@@ -1962,7 +1975,7 @@ QImage QFontEngineFT::alphaMapForGlyph(glyph_t g, QFixed subPixelPosition, const
     if (!img.isNull())
         return img;
 
-    return QFontEngine::alphaMapForGlyph(g);
+    return QFontEngine::alphaMapForGlyph(g, subPixelPosition, t);
 }
 
 QImage QFontEngineFT::alphaRGBMapForGlyph(glyph_t g, QFixed subPixelPosition, const QTransform &t)