Restore smooth-scaled drawing of 0.5x-2.0x scaled glyphs in the GL engine

Commit b5922c89ba942 (Add support for retina glyph-based text drawing
in the GL engine), and commit 155a20628b91 (Use QPaintEngineEx to decide
if font is too big for using the glyph cache) together changed the
behavior of drawing scaled cached glyphs in the GL engine, producing
blocky text drawing when using the FreeType font engine.

Whereas before we would cache all glyphs without any transform, and
let the paint engine take care of the transform, commit b5922c added
support for a scaled GL glyph cache, resulting in a 2x cache, drawn
at 1x for example. The problem was that the FreeType engine claimed
to support producing glyphs at 2x, but did that using the QFontEngine
baseclass implementations, which use a simple fast-transform to scale
up the glyphs. The result was a 2x cache with horrible looking glyphs.

The first step in fixing this issue was to have the FreeType engine
claim to only support translations. This would then make the paint
engine choose path-based drawing for all scaled text, which is slow.

To restore the optimization that we would draw 0.5x-2.0x scaled text
using a smooth-scale in the GL engine (which was removed in 155a206),
we then needed to extend shouldDrawCachedGlyphs() and add a special
condition for this, coupled with a bit of logic in drawCachedGlyphs()
that ensures we don't propagate the painter scale to the glyph cache
if the engine is not able to produce scaled glyphs for it. This
means we get the old behavior of the GL engine doing a smooth scale
of the 1x glyph in the cache.

Finally, since the raster engine also checks if the font engine
supports the current transform, but for FreeType then ends up in
a separate code path when actually drawing the glyphs (as the
FreeType font engine supports internal glyph caching), we need
to add a corresponding check for hasInternalCaching() in the
shouldDrawCachedGlyphs() function, now that the FreeType engine
only reports that it supports translations.

Change-Id: Id03de896dec5f29535b281fb235332ef018045d8
Reviewed-by: Eskil Abrahamsen Blomfeldt <eskil.abrahamsen-blomfeldt@digia.com>

5 files changed, 86 insertions, 26 deletions

diff --git a/src/gui/opengl/qopenglpaintengine.cpp b/src/gui/opengl/qopenglpaintengine.cpp
index 2faf3ffad5..78f5080caf 100644
--- a/src/gui/opengl/qopenglpaintengine.cpp
+++ b/src/gui/opengl/qopenglpaintengine.cpp
@@ -1550,8 +1550,28 @@ namespace {
 
 bool QOpenGL2PaintEngineEx::shouldDrawCachedGlyphs(QFontEngine *fontEngine, const QTransform &t) const
 {
-    // Don't try to cache vastly transformed fonts
-    return t.type() < QTransform::TxProject && QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, t);
+    // The paint engine does not support projected cached glyph drawing
+    if (t.type() == QTransform::TxProject)
+        return false;
+
+    // The font engine might not support filling the glyph cache
+    // with the given transform applied, in which case we need to
+    // fall back to the QPainterPath code-path.
+    if (!fontEngine->supportsTransformation(t)) {
+        // Except that drawing paths is slow, so for scales between
+        // 0.5 and 2.0 we leave the glyph cache untransformed and deal
+        // with the transform ourselves when painting, resulting in
+        // drawing 1x cached glyphs with a smooth-scale.
+        float det = t.determinant();
+        if (det >= 0.25f && det <= 4.f) {
+            // Assuming the baseclass still agrees
+            return QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, t);
+        }
+
+        return false; // Fall back to path-drawing
+    }
+
+    return QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, t);
 }
 
 void QOpenGL2PaintEngineExPrivate::drawCachedGlyphs(QFontEngineGlyphCache::Type glyphType,
@@ -1563,20 +1583,24 @@ void QOpenGL2PaintEngineExPrivate::drawCachedGlyphs(QFontEngineGlyphCache::Type
 
     void *cacheKey = ctx->shareGroup();
     bool recreateVertexArrays = false;
-    QFontEngine *fe = staticTextItem->fontEngine();
 
-    // We allow scaling, so that the glyph-cache will contain glyphs with the
-    // appropriate resolution in the case of displays with a device-pixel-ratio != 1.
-    QTransform transform = s->matrix.type() < QTransform::TxRotate ?
-        QTransform::fromScale(qAbs(s->matrix.m11()), qAbs(s->matrix.m22())) :
-        QTransform::fromScale(
-            QVector2D(s->matrix.m11(), s->matrix.m12()).length(),
-            QVector2D(s->matrix.m21(), s->matrix.m22()).length());
+    QTransform glyphCacheTransform;
+    QFontEngine *fe = staticTextItem->fontEngine();
+    if (fe->supportsTransformation(s->matrix)) {
+        // The font-engine supports rendering glyphs with the current transform, so we
+        // build a glyph-cache with the scale pre-applied, so that the cache contains
+        // glyphs with the appropriate resolution in the case of retina displays.
+        glyphCacheTransform = s->matrix.type() < QTransform::TxRotate ?
+            QTransform::fromScale(qAbs(s->matrix.m11()), qAbs(s->matrix.m22())) :
+            QTransform::fromScale(
+                QVector2D(s->matrix.m11(), s->matrix.m12()).length(),
+                QVector2D(s->matrix.m21(), s->matrix.m22()).length());
+    }
 
     QOpenGLTextureGlyphCache *cache =
-            (QOpenGLTextureGlyphCache *) fe->glyphCache(cacheKey, glyphType, transform);
+            (QOpenGLTextureGlyphCache *) fe->glyphCache(cacheKey, glyphType, glyphCacheTransform);
     if (!cache || cache->cacheType() != glyphType || cache->contextGroup() == 0) {
-        cache = new QOpenGLTextureGlyphCache(glyphType, transform);
+        cache = new QOpenGLTextureGlyphCache(glyphType, glyphCacheTransform);
         fe->setGlyphCache(cacheKey, cache);
         recreateVertexArrays = true;
     }
diff --git a/src/gui/painting/qpaintengine_raster.cpp b/src/gui/painting/qpaintengine_raster.cpp
index fb3328090b..d1e9b81faa 100644
--- a/src/gui/painting/qpaintengine_raster.cpp
+++ b/src/gui/painting/qpaintengine_raster.cpp
@@ -3333,8 +3333,10 @@ bool QRasterPaintEngine::shouldDrawCachedGlyphs(QFontEngine *fontEngine, const Q
 
     // The font engine might not support filling the glyph cache
     // with the given transform applied, in which case we need to
-    // fall back to the QPainterPath code-path.
-    if (!fontEngine->supportsTransformation(m))
+    // fall back to the QPainterPath code-path. This does not apply
+    // for engines with internal caching, as we don't use the engine
+    // to fill up our cache in that case.
+    if (!fontEngine->hasInternalCaching() && !fontEngine->supportsTransformation(m))
         return false;
 
     return QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, m);
diff --git a/src/gui/text/qfontengine_ft.cpp b/src/gui/text/qfontengine_ft.cpp
index 4545645dc6..7b4925a9c8 100644
--- a/src/gui/text/qfontengine_ft.cpp
+++ b/src/gui/text/qfontengine_ft.cpp
@@ -1437,6 +1437,14 @@ void QFontEngineFT::getUnscaledGlyph(glyph_t glyph, QPainterPath *path, glyph_me
     unlockFace();
 }
 
+bool QFontEngineFT::supportsTransformation(const QTransform &transform) const
+{
+    // The freetype engine falls back to QFontEngine for tranformed glyphs,
+    // which uses fast-tranform and produces very ugly results, so we claim
+    // to support just translations.
+    return transform.type() <= QTransform::TxTranslate;
+}
+
 static inline unsigned int getChar(const QChar *str, int &i, const int len)
 {
     uint ucs4 = str[i].unicode();
diff --git a/src/gui/text/qfontengine_ft_p.h b/src/gui/text/qfontengine_ft_p.h
index e09fa6f94f..bd4c855b91 100644
--- a/src/gui/text/qfontengine_ft_p.h
+++ b/src/gui/text/qfontengine_ft_p.h
@@ -233,6 +233,8 @@ private:
 
     virtual void getUnscaledGlyph(glyph_t glyph, QPainterPath *path, glyph_metrics_t *metrics);
 
+    virtual bool supportsTransformation(const QTransform &transform) const;
+
     virtual bool canRender(const QChar *string, int len);
 
     virtual void addGlyphsToPath(glyph_t *glyphs, QFixedPoint *positions, int nglyphs,
diff --git a/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp b/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
index 824e7ecbad..80660041b7 100644
--- a/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
+++ b/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
@@ -627,8 +627,28 @@ bool QGL2PaintEngineEx::isNativePaintingActive() const {
 
 bool QGL2PaintEngineEx::shouldDrawCachedGlyphs(QFontEngine *fontEngine, const QTransform &t) const
 {
-    // Don't try to cache vastly transformed fonts
-    return t.type() < QTransform::TxProject && QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, t);
+    // The paint engine does not support projected cached glyph drawing
+    if (t.type() == QTransform::TxProject)
+        return false;
+
+    // The font engine might not support filling the glyph cache
+    // with the given transform applied, in which case we need to
+    // fall back to the QPainterPath code-path.
+    if (!fontEngine->supportsTransformation(t)) {
+        // Except that drawing paths is slow, so for scales between
+        // 0.5 and 2.0 we leave the glyph cache untransformed and deal
+        // with the transform ourselves when painting, resulting in
+        // drawing 1x cached glyphs with a smooth-scale.
+        float det = t.determinant();
+        if (det >= 0.25f && det <= 4.f) {
+            // Assuming the baseclass still agrees
+            return QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, t);
+        }
+
+        return false; // Fall back to path-drawing
+    }
+
+    return QPaintEngineEx::shouldDrawCachedGlyphs(fontEngine, t);
 }
 
 void QGL2PaintEngineExPrivate::transferMode(EngineMode newMode)
@@ -1586,19 +1606,23 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(QFontEngineGlyphCache::Type glyp
     void *cacheKey = const_cast<QGLContext *>(QGLContextPrivate::contextGroup(ctx)->context());
     bool recreateVertexArrays = false;
 
-    // We allow scaling, so that the glyph-cache will contain glyphs with the
-    // appropriate resolution in the case of displays with a device-pixel-ratio != 1.
-    QTransform transform = s->matrix.type() < QTransform::TxRotate ?
-        QTransform::fromScale(qAbs(s->matrix.m11()), qAbs(s->matrix.m22())) :
-        QTransform::fromScale(
-            QVector2D(s->matrix.m11(), s->matrix.m12()).length(),
-            QVector2D(s->matrix.m21(), s->matrix.m22()).length());
-
+    QTransform glyphCacheTransform;
     QFontEngine *fe = staticTextItem->fontEngine();
+    if (fe->supportsTransformation(s->matrix)) {
+        // The font-engine supports rendering glyphs with the current transform, so we
+        // build a glyph-cache with the scale pre-applied, so that the cache contains
+        // glyphs with the appropriate resolution in the case of retina displays.
+        glyphCacheTransform = s->matrix.type() < QTransform::TxRotate ?
+            QTransform::fromScale(qAbs(s->matrix.m11()), qAbs(s->matrix.m22())) :
+            QTransform::fromScale(
+                QVector2D(s->matrix.m11(), s->matrix.m12()).length(),
+                QVector2D(s->matrix.m21(), s->matrix.m22()).length());
+    }
+
     QGLTextureGlyphCache *cache =
-            (QGLTextureGlyphCache *) fe->glyphCache(cacheKey, glyphType, transform);
+            (QGLTextureGlyphCache *) fe->glyphCache(cacheKey, glyphType, glyphCacheTransform);
     if (!cache || cache->cacheType() != glyphType || cache->contextGroup() == 0) {
-        cache = new QGLTextureGlyphCache(glyphType, transform);
+        cache = new QGLTextureGlyphCache(glyphType, glyphCacheTransform);
         fe->setGlyphCache(cacheKey, cache);
         recreateVertexArrays = true;
     }