CoreText: Add font antialiasing and smoothing helper functions

The font smoothing helper in particular now takes into account whether
or not we're dealing with color glyphs, in which case we shouldn't (or
don't need to) smooth, and also makes sure the QFont::NoSubpixelAntialias
style strategy doesn't affect font smoothing when we're dealing with
non-subpixel-antialiased font smoothing, as on macOS 10.14.

Change-Id: Ibd477158629402c55cafec31576b6d9901d184cf
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>

2 files changed, 27 insertions, 4 deletions

diff --git a/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm b/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
index 7a26416868..6c2ffba92e 100644
--- a/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
+++ b/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
@@ -712,6 +712,28 @@ QCoreTextFontEngine::FontSmoothing QCoreTextFontEngine::fontSmoothing()
     return cachedFontSmoothing;
 }
 
+bool QCoreTextFontEngine::shouldAntialias() const
+{
+    return !(fontDef.styleStrategy & QFont::NoAntialias);
+}
+
+bool QCoreTextFontEngine::shouldSmoothFont() const
+{
+    if (hasColorGlyphs())
+        return false;
+
+    if (!shouldAntialias())
+        return false;
+
+    switch (fontSmoothing()) {
+    case Disabled: return false;
+    case Subpixel: return !(fontDef.styleStrategy & QFont::NoSubpixelAntialias);
+    case Grayscale: return true;
+    }
+
+    Q_UNREACHABLE();
+}
+
 bool QCoreTextFontEngine::expectsGammaCorrectedBlending() const
 {
     // Only works well when font-smoothing is enabled
@@ -755,10 +777,9 @@ QImage QCoreTextFontEngine::imageForGlyph(glyph_t glyph, QFixed subPixelPosition
                                              qt_mac_bitmapInfoForImage(im));
     Q_ASSERT(ctx);
     CGContextSetFontSize(ctx, fontDef.pixelSize);
-    const bool antialias = !(fontDef.styleStrategy & QFont::NoAntialias);
-    CGContextSetShouldAntialias(ctx, antialias);
-    const bool smoothing = antialias && !(fontDef.styleStrategy & QFont::NoSubpixelAntialias);
-    CGContextSetShouldSmoothFonts(ctx, smoothing);
+
+    CGContextSetShouldAntialias(ctx, shouldAntialias());
+    CGContextSetShouldSmoothFonts(ctx, shouldSmoothFont());
 
     CGAffineTransform cgMatrix = CGAffineTransformIdentity;
 
diff --git a/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h b/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h
index 0ff428084f..28810f13bc 100644
--- a/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h
+++ b/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h
@@ -139,6 +139,8 @@ protected:
     QImage imageForGlyph(glyph_t glyph, QFixed subPixelPosition, const QTransform &m);
     void loadAdvancesForGlyphs(QVarLengthArray<CGGlyph> &cgGlyphs, QGlyphLayout *glyphs) const;
     bool hasColorGlyphs() const;
+    bool shouldAntialias() const;
+    bool shouldSmoothFont() const;
 
     QCFType<CTFontRef> ctfont;
     QCFType<CGFontRef> cgFont;