Update clipping area when antialiasing changes

Antialiasing is disabled when the painter's antialiasing attribute
is set behind the clipping function(example `setClipPath` or
`setClipRegion`). The cause of this problem is that the
antialiasing state of the clipping region is not updated after the
antialiasing attribute is set.

A variable is required to record the painter's transformation state
set before the clipping function, because the transformation will be
applied to the clipping region, resulting in the abnormal clipping
region. The value of `s->matrix` is not accurate for the clipping fun-
ction.

Pick-to: 6.2
Fixes: QTBUG-97269
Change-Id: I409a9db32efc3b991ebb97ec9aed19bbddb273d8
Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io>

4 files changed, 117 insertions, 0 deletions

diff --git a/src/gui/painting/qpaintengine_raster.cpp b/src/gui/painting/qpaintengine_raster.cpp
index 76489da0b4..845291b61b 100644
--- a/src/gui/painting/qpaintengine_raster.cpp
+++ b/src/gui/painting/qpaintengine_raster.cpp
@@ -882,6 +882,9 @@ void QRasterPaintEngine::renderHintsChanged()
 
     Q_D(QRasterPaintEngine);
     d->recalculateFastImages();
+
+    if (was_aa != s->flags.antialiased)
+        d->updateClipping();
 }
 
 /*!
@@ -1245,6 +1248,7 @@ void QRasterPaintEngine::clip(const QVectorPath &path, Qt::ClipOperation op)
         ensureOutlineMapper();
         d->rasterize(d->outlineMapper->convertPath(path), qt_span_clip, &clipData, nullptr);
 
+        newClip->clipTransform = s->matrix;
         newClip->fixup();
 
         if (s->flags.has_clip_ownership)
@@ -1329,6 +1333,7 @@ bool QRasterPaintEngine::setClipRectInDeviceCoords(const QRect &r, Qt::ClipOpera
         return false;
     }
 
+    s->clip->clipTransform = s->matrix;
     qrasterpaintengine_dirty_clip(d, s);
     return true;
 }
@@ -1384,6 +1389,7 @@ void QRasterPaintEngine::clip(const QRegion &region, Qt::ClipOperation op)
         else if (curClip->hasRegionClip)
             newClip->setClipRegion(r & curClip->clipRegion);
 
+        newClip->clipTransform = s->matrix;
         qrasterpaintengine_dirty_clip(d, s);
     }
 }
@@ -3701,6 +3707,37 @@ void QRasterPaintEnginePrivate::rasterize(QT_FT_Outline *outline,
     free(rasterPoolOnHeap);
 }
 
+void QRasterPaintEnginePrivate::updateClipping()
+{
+    Q_Q(QRasterPaintEngine);
+    QRasterPaintEngineState *s = q->state();
+
+    if (!s->clipEnabled)
+        return;
+
+    bool noClipPath = s->clipPath.isEmpty();
+    bool noClipRegion = s->clipRegion.isEmpty();
+
+    if (noClipPath && noClipRegion)
+        return;
+
+    if (!s->clip)
+        return;
+
+    const QTransform stateTransform = s->matrix;
+    s->matrix = s->clip->clipTransform;
+
+    qrasterpaintengine_state_setNoClip(s);
+
+    if (noClipRegion) {
+        q->clip(qtVectorPathForPath(s->clipPath), s->clipOperation);
+    } else {
+        q->clip(s->clipRegion, s->clipOperation);
+    }
+
+    s->matrix = stateTransform;
+}
+
 void QRasterPaintEnginePrivate::recalculateFastImages()
 {
     Q_Q(QRasterPaintEngine);
diff --git a/src/gui/painting/qpaintengine_raster_p.h b/src/gui/painting/qpaintengine_raster_p.h
index d25fe4c3b3..9751eb3da8 100644
--- a/src/gui/painting/qpaintengine_raster_p.h
+++ b/src/gui/painting/qpaintengine_raster_p.h
@@ -276,6 +276,7 @@ public:
     void rasterize(QT_FT_Outline *outline, ProcessSpans callback, QSpanData *spanData, QRasterBuffer *rasterBuffer);
     void rasterize(QT_FT_Outline *outline, ProcessSpans callback, void *userData, QRasterBuffer *rasterBuffer);
     void updateMatrixData(QSpanData *spanData, const QBrush &brush, const QTransform &brushMatrix);
+    void updateClipping();
 
     void systemStateChanged() override;
 
@@ -374,6 +375,7 @@ public:
 
     QRect clipRect;
     QRegion clipRegion;
+    QTransform clipTransform;
 
     uint enabled : 1;
     uint hasRectClip : 1;
diff --git a/src/gui/painting/qpainter.cpp b/src/gui/painting/qpainter.cpp
index e5caf0813b..0ebc6253db 100644
--- a/src/gui/painting/qpainter.cpp
+++ b/src/gui/painting/qpainter.cpp
@@ -2844,6 +2844,7 @@ void QPainter::setClipRegion(const QRegion &r, Qt::ClipOperation op)
             d->state->clipInfo.clear();
         d->state->clipInfo.append(QPainterClipInfo(r, op, d->state->matrix));
         d->state->clipOperation = op;
+        d->state->clipRegion = r;
         return;
     }
 
@@ -3057,6 +3058,7 @@ void QPainter::setClipPath(const QPainterPath &path, Qt::ClipOperation op)
             d->state->clipInfo.clear();
         d->state->clipInfo.append(QPainterClipInfo(path, op, d->state->matrix));
         d->state->clipOperation = op;
+        d->state->clipPath = path;
         return;
     }
 
diff --git a/tests/auto/other/lancelot/scripts/clippath_antialiasing.qps b/tests/auto/other/lancelot/scripts/clippath_antialiasing.qps
new file mode 100644
index 0000000000..193ce8b7ed
--- /dev/null
+++ b/tests/auto/other/lancelot/scripts/clippath_antialiasing.qps
@@ -0,0 +1,76 @@
+# There was no serration in either case
+save
+setBrush black SolidPattern
+drawRect 0.0 0.0 500.0 300.0
+restore
+
+save
+setRenderHint Antialiasing true
+path_addEllipse mypath1 10.0 10.0 200.0 200.0
+path_addRect mypath1 10.0 210.0 200.0 40.0
+path_addPolygon mypath1 [ 10 250  50 280  90 300 130 300  170 280  210 250]
+
+setClipPath mypath1 ReplaceClip
+setPen NoPen
+setBrush cyan SolidPattern
+drawRect 10.0 10.0 400.0 400.0
+restore
+save
+setRenderHint Antialiasing false
+path_addEllipse mypath2 220.0 10.0 200.0 200.0
+path_addRect mypath2 220.0 210.0 200.0 40.0
+path_addPolygon mypath2 [ 220 250  270 280  300 300 340 300  380 280  420 250]
+setClipPath mypath2 ReplaceClip
+setRenderHint Antialiasing true
+setPen NoPen
+setBrush cyan SolidPattern
+drawRect 220.0 10.0 300.0 300.0
+restore
+
+setPen red
+setBrush NoBrush
+drawText 20 250 "Antialiasing before setClipPath"
+drawText 240 250 "Antialiasing after setClipPath"
+
+# Test that the clipping region is not abnormal after some transformations
+translate 100 300
+
+save
+path_addEllipse mypath3 10.0 10.0 200.0 200.0
+path_addRect mypath3 10.0 210.0 200.0 40.0
+setPen black SolidLine
+setBrush NoBrush
+setRenderHint Antialiasing true
+drawPath mypath3
+setClipPath mypath3 ReplaceClip
+rotate 60
+setFont "times" 10 Bold
+drawText 10 70 "Antialiasing before setClipPath - Transformation"
+drawText 10 40 "Antialiasing before setClipPath - Transformation"
+drawText 10 10 "Antialiasing before setClipPath - Transformation"
+drawText 10 -20 "Antialiasing before setClipPath - Transformation"
+drawText 10 -50 "Antialiasing before setClipPath - Transformation"
+drawText 10 -80 "Antialiasing before setClipPath - Transformation"
+restore
+
+translate 0 250
+
+save
+path_addEllipse mypath4 10.0 10.0 200.0 200.0
+path_addRect mypath4 10.0 210.0 200.0 40.0
+setPen black SolidLine
+setBrush NoBrush
+drawPath mypath4
+setClipPath mypath4 ReplaceClip
+setRenderHint Antialiasing true
+rotate 60
+setFont "times" 10 Bold
+drawText 10 70 "Antialiasing after setClipPath - Transformation"
+drawText 10 40 "Antialiasing after setClipPath - Transformation"
+drawText 10 10 "Antialiasing after setClipPath - Transformation"
+drawText 10 -20 "Antialiasing after setClipPath - Transformation"
+drawText 10 -50 "Antialiasing after setClipPath - Transformation"
+drawText 10 -80 "Antialiasing after setClipPath - Transformation"
+restore
+
+