CoreText: Modernize font smoothing and antialiasing threshold detection

The way macOS does font smoothing has changed in Mojave, and we need to
take both this new algorithm into account, as well as support users who
set legacy preferences to revert back to subpixel font smoothing.

As a followup to this patch we will tweak some of the existing logic
to take the new font smoothing algorithm into account, so this is
just a first step.

Change-Id: If37014c18515f406b8bb8194c9df7a75c2eb10fc
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>

3 files changed, 131 insertions, 74 deletions

diff --git a/src/platformsupport/fontdatabases/mac/qcoretextfontdatabase.mm b/src/platformsupport/fontdatabases/mac/qcoretextfontdatabase.mm
index 3718ebdda6..ba23271e55 100644
--- a/src/platformsupport/fontdatabases/mac/qcoretextfontdatabase.mm
+++ b/src/platformsupport/fontdatabases/mac/qcoretextfontdatabase.mm
@@ -117,71 +117,6 @@ static NSInteger languageMapSort(id obj1, id obj2, void *context)
 QCoreTextFontDatabase::QCoreTextFontDatabase()
     : m_hasPopulatedAliases(false)
 {
-#ifdef Q_OS_MACX
-    /*
-        font_smoothing = 0 means no smoothing, while 1-3 means subpixel
-        antialiasing with different hinting styles (but we don't care about the
-        exact value, only if subpixel rendering is available or not)
-    */
-    int font_smoothing = 0;
-
-#if QT_CONFIG(settings)
-    QSettings appleSettings(QLatin1String("apple.com"));
-    QVariant appleValue = appleSettings.value(QLatin1String("AppleAntiAliasingThreshold"));
-    if (appleValue.isValid())
-        QCoreTextFontEngine::antialiasingThreshold = appleValue.toInt();
-
-    appleValue = appleSettings.value(QLatin1String("AppleFontSmoothing"));
-    if (appleValue.isValid()) {
-        font_smoothing = appleValue.toInt();
-    } else
-#endif // settings
-    {
-        // non-Apple displays do not provide enough information about subpixel rendering so
-        // draw text with cocoa and compare pixel colors to see if subpixel rendering is enabled
-        int w = 10;
-        int h = 10;
-        NSRect rect = NSMakeRect(0.0, 0.0, w, h);
-        NSImage *fontImage = [[NSImage alloc] initWithSize:NSMakeSize(w, h)];
-
-        [fontImage lockFocus];
-
-        [[NSColor whiteColor] setFill];
-        NSRectFill(rect);
-
-        NSString *str = @"X\\";
-        NSFont *font = [NSFont fontWithName:@"Helvetica" size:10.0];
-        NSMutableDictionary *attrs = [NSMutableDictionary dictionary];
-        [attrs setObject:font forKey:NSFontAttributeName];
-        [attrs setObject:[NSColor blackColor] forKey:NSForegroundColorAttributeName];
-
-        [str drawInRect:rect withAttributes:attrs];
-
-        NSBitmapImageRep *nsBitmapImage = [[NSBitmapImageRep alloc] initWithFocusedViewRect:rect];
-
-        [fontImage unlockFocus];
-
-        float red, green, blue;
-        for (int x = 0; x < w; x++) {
-            for (int y = 0; y < h; y++) {
-                NSColor *pixelColor = [nsBitmapImage colorAtX:x y:y];
-                red = [pixelColor redComponent];
-                green = [pixelColor greenComponent];
-                blue = [pixelColor blueComponent];
-                if (red != green || red != blue)
-                    font_smoothing = 1;
-            }
-        }
-
-        [nsBitmapImage release];
-        [fontImage release];
-    }
-    QCoreTextFontEngine::defaultGlyphFormat = (font_smoothing > 0
-                                               ? QFontEngine::Format_A32
-                                               : QFontEngine::Format_A8);
-#else
-    QCoreTextFontEngine::defaultGlyphFormat = QFontEngine::Format_A8;
-#endif
 }
 
 QCoreTextFontDatabase::~QCoreTextFontDatabase()
diff --git a/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm b/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
index e4ee0c0ac4..d939ee1b24 100644
--- a/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
+++ b/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
@@ -85,6 +85,8 @@
 
 QT_BEGIN_NAMESPACE
 
+Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts")
+
 static float SYNTHETIC_ITALIC_SKEW = std::tan(14.f * std::acos(0.f) / 90.f);
 
 bool QCoreTextFontEngine::ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length)
@@ -133,9 +135,6 @@ QFont::Weight QCoreTextFontEngine::qtWeightFromCFWeight(float value)
     return ret;
 }
 
-int QCoreTextFontEngine::antialiasingThreshold = 0;
-QFontEngine::GlyphFormat QCoreTextFontEngine::defaultGlyphFormat = QFontEngine::Format_A32;
-
 CGAffineTransform qt_transform_from_fontdef(const QFontDef &fontDef)
 {
     CGAffineTransform transform = CGAffineTransformIdentity;
@@ -236,8 +235,10 @@ void QCoreTextFontEngine::init()
 
     if (traits & kCTFontColorGlyphsTrait)
         glyphFormat = QFontEngine::Format_ARGB;
+    else if (fontSmoothing() == FontSmoothing::Subpixel)
+        glyphFormat = QFontEngine::Format_A32;
     else
-        glyphFormat = defaultGlyphFormat;
+        glyphFormat = QFontEngine::Format_A8;
 
     if (traits & kCTFontItalicTrait)
         fontDef.style = QFont::StyleItalic;
@@ -520,7 +521,7 @@ static void convertCGPathToQPainterPath(void *info, const CGPathElement *element
             myInfo->path->closeSubpath();
             break;
         default:
-            qDebug() << "Unhandled path transform type: " << element->type;
+            qCWarning(lcQpaFonts) << "Unhandled path transform type: " << element->type;
     }
 
 }
@@ -608,6 +609,118 @@ glyph_metrics_t QCoreTextFontEngine::alphaMapBoundingBox(glyph_t glyph, QFixed s
     return br;
 }
 
+int QCoreTextFontEngine::antialiasingThreshold()
+{
+    static const int antialiasingThreshold = [] {
+        auto defaults = [NSUserDefaults standardUserDefaults];
+        int threshold = [defaults integerForKey:@"AppleAntiAliasingThreshold"];
+        qCDebug(lcQpaFonts) << "Resolved antialiasing threshold. Defaults ="
+            << [[defaults dictionaryRepresentation] dictionaryWithValuesForKeys:@[
+                @"AppleAntiAliasingThreshold"
+            ]] << "Result =" << threshold;
+        return threshold;
+    }();
+    return antialiasingThreshold;
+}
+
+/*
+    Apple has gone through many iterations of its font smoothing algorithms,
+    and there are many ways to enable or disable certain aspects of it. As
+    keeping up with all the different toggles and behavior differences between
+    macOS versions is tricky, we resort to rendering a single glyph in a few
+    configurations, picking up the font smoothing algorithm from the observed
+    result.
+
+    The possible values are:
+
+     - Disabled: No font smoothing is applied.
+
+       Possibly triggered by the user unchecking the "Use font smoothing when
+       available" checkbox in the system preferences or setting AppleFontSmoothing
+       to 0. Also controlled by the CGContextSetAllowsFontSmoothing() API,
+       which gets its default from the settings above. This API overrides
+       the more granular CGContextSetShouldSmoothFonts(), which we use to
+       enable (request) or disable font smoothing.
+
+       Note that this does not exclude normal antialiasing, controlled by
+       the CGContextSetShouldAntialias() API.
+
+     - Subpixel: Font smoothing is applied, and affects subpixels.
+
+       This was the default mode on macOS versions prior to 10.14 (Mojave).
+       The font dilation (stem darkening) parameters were controlled by the
+       AppleFontSmoothing setting, ranging from 1 to 3 (light to strong).
+
+       On Mojave it is no longer supported, but can be triggered by a legacy
+       override (CGFontRenderingFontSmoothingDisabled=NO), so we need to
+       still account for it, otherwise users will have a bad time.
+
+     - Grayscale: Font smoothing is applied, but does not affect subpixels.
+
+       This is the default mode on macOS 10.14 (Mojave). The font dilation
+       (stem darkening) parameters are not affected by the AppleFontSmoothing
+       setting, but are instead computed based on the fill color used when
+       drawing the glyphs (white fill gives a lighter dilation than black
+       fill). This affects how we build our glyph cache, since we produce
+       alpha maps by drawing white on black.
+*/
+QCoreTextFontEngine::FontSmoothing QCoreTextFontEngine::fontSmoothing()
+{
+    static const FontSmoothing cachedFontSmoothing = [] {
+        static const int kSize = 10;
+        QCFType<CTFontRef> font = CTFontCreateWithName(CFSTR("Helvetica"), kSize, nullptr);
+
+        UniChar character('X'); CGGlyph glyph;
+        CTFontGetGlyphsForCharacters(font, &character, &glyph, 1);
+
+        auto drawGlyph = [&](bool smooth) -> QImage {
+            QImage image(kSize, kSize, QImage::Format_RGB32);
+            image.fill(0);
+
+            QMacCGContext ctx(&image);
+            CGContextSetTextDrawingMode(ctx, kCGTextFill);
+            CGContextSetGrayFillColor(ctx, 1, 1);
+
+            // Will be ignored if CGContextSetAllowsFontSmoothing() has been
+            // set to false by CoreGraphics based on user defaults.
+            CGContextSetShouldSmoothFonts(ctx, smooth);
+
+            CTFontDrawGlyphs(font, &glyph, &CGPointZero, 1, ctx);
+            return image;
+        };
+
+        QImage nonSmoothed = drawGlyph(false);
+        QImage smoothed = drawGlyph(true);
+
+        FontSmoothing fontSmoothing = FontSmoothing::Disabled;
+        [&] {
+            for (int x = 0; x < kSize; ++x) {
+                for (int y = 0; y < kSize; ++y) {
+                    QRgb sp = smoothed.pixel(x, y);
+                    if (qRed(sp) != qGreen(sp) || qRed(sp) != qBlue(sp)) {
+                        fontSmoothing = FontSmoothing::Subpixel;
+                        return;
+                    }
+
+                    if (sp != nonSmoothed.pixel(x, y))
+                        fontSmoothing = FontSmoothing::Grayscale;
+                }
+            }
+        }();
+
+        auto defaults = [NSUserDefaults standardUserDefaults];
+        qCDebug(lcQpaFonts) << "Resolved font smoothing algorithm. Defaults ="
+            << [[defaults dictionaryRepresentation] dictionaryWithValuesForKeys:@[
+                @"AppleFontSmoothing",
+                @"CGFontRenderingFontSmoothingDisabled"
+            ]] << "Result =" << fontSmoothing;
+
+        return fontSmoothing;
+    }();
+
+    return cachedFontSmoothing;
+}
+
 bool QCoreTextFontEngine::expectsGammaCorrectedBlending() const
 {
     // Only works well when font-smoothing is enabled
@@ -652,7 +765,7 @@ QImage QCoreTextFontEngine::imageForGlyph(glyph_t glyph, QFixed subPixelPosition
                                              qt_mac_bitmapInfoForImage(im));
     Q_ASSERT(ctx);
     CGContextSetFontSize(ctx, fontDef.pixelSize);
-    const bool antialias = (aa || fontDef.pointSize > antialiasingThreshold) && !(fontDef.styleStrategy & QFont::NoAntialias);
+    const bool antialias = (aa || fontDef.pointSize > antialiasingThreshold()) && !(fontDef.styleStrategy & QFont::NoAntialias);
     CGContextSetShouldAntialias(ctx, antialias);
     const bool smoothing = antialias && !(fontDef.styleStrategy & QFont::NoSubpixelAntialias);
     CGContextSetShouldSmoothFonts(ctx, smoothing);
diff --git a/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h b/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h
index 7ed2faff8e..2ce46a4706 100644
--- a/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h
+++ b/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h
@@ -53,6 +53,7 @@
 
 #include <private/qfontengine_p.h>
 #include <private/qcore_mac_p.h>
+#include <QtCore/qloggingcategory.h>
 
 #ifdef Q_OS_OSX
 #include <ApplicationServices/ApplicationServices.h>
@@ -63,8 +64,12 @@
 
 QT_BEGIN_NAMESPACE
 
+Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts)
+
 class QCoreTextFontEngine : public QFontEngine
 {
+    Q_GADGET
+
 public:
     QCoreTextFontEngine(CTFontRef font, const QFontDef &def);
     QCoreTextFontEngine(CGFontRef font, const QFontDef &def);
@@ -118,13 +123,17 @@ public:
 
     QFontEngine::Properties properties() const override;
 
+    enum FontSmoothing { Disabled, Subpixel, Grayscale };
+    Q_ENUM(FontSmoothing);
+
+    static FontSmoothing fontSmoothing();
+    static int antialiasingThreshold();
+
     static bool ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length);
     static QFont::Weight qtWeightFromCFWeight(float value);
 
-    static int antialiasingThreshold;
-    static QFontEngine::GlyphFormat defaultGlyphFormat;
-
     static QCoreTextFontEngine *create(const QByteArray &fontData, qreal pixelSize, QFont::HintingPreference hintingPreference);
+
 protected:
     QCoreTextFontEngine(const QFontDef &def);
     void init();