From 5c351da046992f745d63d86be6d45b3620b7ce95 Mon Sep 17 00:00:00 2001 From: Allan Sandfeld Jensen Date: Fri, 28 Jun 2019 12:28:38 +0200 Subject: Protect against integer overflow in painting transformed images Makes it safe to sample pixel coordinates above 32767. Fixes: QTBUG-76829 Change-Id: I5965afef1bd65106fcfc130dd37572309eacbe42 Reviewed-by: Friedemann Kleint Reviewed-by: Eirik Aavitsland --- src/gui/painting/qdrawhelper.cpp | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/gui/painting/qdrawhelper.cpp b/src/gui/painting/qdrawhelper.cpp index 1f7ab5006c..6265d51037 100644 --- a/src/gui/painting/qdrawhelper.cpp +++ b/src/gui/painting/qdrawhelper.cpp @@ -1991,6 +1991,23 @@ inline void fetchTransformed_pixelBounds(int max, int l1, int l2, int &v) } } +static inline bool canUseFastMatrixPath(const qreal cx, const qreal cy, const qsizetype length, const QSpanData *data) +{ + if (Q_UNLIKELY(!data->fast_matrix)) + return false; + + qreal fx = (data->m21 * cy + data->m11 * cx + data->dx) * fixed_scale; + qreal fy = (data->m22 * cy + data->m12 * cx + data->dy) * fixed_scale; + qreal minc = std::min(fx, fy); + qreal maxc = std::max(fx, fy); + fx += std::trunc(data->m11 * fixed_scale) * length; + fy += std::trunc(data->m12 * fixed_scale) * length; + minc = std::min(minc, std::min(fx, fy)); + maxc = std::max(maxc, std::max(fx, fy)); + + return minc >= std::numeric_limits::min() && maxc <= std::numeric_limits::max(); +} + template static void QT_FASTCALL fetchTransformed_fetcher(T *buffer, const QSpanData *data, int y, int x, int length) @@ -2008,7 +2025,7 @@ static void QT_FASTCALL fetchTransformed_fetcher(T *buffer, const QSpanData *dat // When templated 'fetch' should be inlined at compile time: const FetchPixelFunc fetch = (bpp == QPixelLayout::BPPNone) ? qFetchPixel[layout->bpp] : FetchPixelFunc(fetchPixel); - if (data->fast_matrix) { + if (canUseFastMatrixPath(cx, cy, length, data)) { // The increment pr x in the scanline int fdx = (int)(data->m11 * fixed_scale); int fdy = (int)(data->m12 * fixed_scale); @@ -2962,7 +2979,7 @@ static const uint * QT_FASTCALL fetchTransformedBilinearARGB32PM(uint *buffer, c uint *end = buffer + length; uint *b = buffer; - if (data->fast_matrix) { + if (canUseFastMatrixPath(cx, cy, length, data)) { // The increment pr x in the scanline int fdx = (int)(data->m11 * fixed_scale); int fdy = (int)(data->m12 * fixed_scale); @@ -3319,7 +3336,7 @@ static const uint *QT_FASTCALL fetchTransformedBilinear(uint *buffer, const Oper const qreal cx = x + qreal(0.5); const qreal cy = y + qreal(0.5); - if (data->fast_matrix) { + if (canUseFastMatrixPath(cx, cy, length, data)) { // The increment pr x in the scanline int fdx = (int)(data->m11 * fixed_scale); int fdy = (int)(data->m12 * fixed_scale); @@ -3505,7 +3522,7 @@ static const QRgba64 *QT_FASTCALL fetchTransformedBilinear64_uint32(QRgba64 *buf QRgba64 *end = buffer + length; QRgba64 *b = buffer; - if (data->fast_matrix) { + if (canUseFastMatrixPath(cx, cy, length, data)) { // The increment pr x in the scanline const int fdx = (int)(data->m11 * fixed_scale); const int fdy = (int)(data->m12 * fixed_scale); @@ -3663,7 +3680,7 @@ static const QRgba64 *QT_FASTCALL fetchTransformedBilinear64_uint64(QRgba64 *buf QRgba64 *end = buffer + length; QRgba64 *b = buffer; - if (data->fast_matrix) { + if (canUseFastMatrixPath(cx, cy, length, data)) { // The increment pr x in the scanline const int fdx = (int)(data->m11 * fixed_scale); const int fdy = (int)(data->m12 * fixed_scale); -- cgit v1.2.3