Avoid int overflow in QImage rotate90/180/270

Fixes: QTBUG-91223
Change-Id: Ice53c80d695a5ffdf9162df84e7c9b1e43106bae
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
(cherry picked from commit 8daa94431341afece6beb052e6224d215f8507b7)

1 files changed, 16 insertions, 11 deletions

diff --git a/src/gui/painting/qmemrotate.cpp b/src/gui/painting/qmemrotate.cpp
index 43aeff3268..a60d373732 100644
--- a/src/gui/painting/qmemrotate.cpp
+++ b/src/gui/painting/qmemrotate.cpp
@@ -45,10 +45,10 @@ static const int tileSize = 32;
 
 template <class T>
 Q_STATIC_TEMPLATE_FUNCTION
-inline void qt_memrotate90_tiled(const T *src, int w, int h, int sstride, T *dest, int dstride)
+inline void qt_memrotate90_tiled(const T *src, int w, int h, int isstride, T *dest, int idstride)
 {
-    sstride /= sizeof(T);
-    dstride /= sizeof(T);
+    const qsizetype sstride = isstride / sizeof(T);
+    const qsizetype dstride = idstride / sizeof(T);
 
     const int pack = sizeof(quint32) / sizeof(T);
     const int unaligned =
@@ -104,9 +104,10 @@ inline void qt_memrotate90_tiled(const T *src, int w, int h, int sstride, T *des
 
 template <class T>
 Q_STATIC_TEMPLATE_FUNCTION
-inline void qt_memrotate90_tiled_unpacked(const T *src, int w, int h, int sstride, T *dest,
-                                          int dstride)
+inline void qt_memrotate90_tiled_unpacked(const T *src, int w, int h, int isstride, T *dest, int idstride)
 {
+    const qsizetype sstride = isstride;
+    const qsizetype dstride = idstride;
     const int numTilesX = (w + tileSize - 1) / tileSize;
     const int numTilesY = (h + tileSize - 1) / tileSize;
 
@@ -132,10 +133,10 @@ inline void qt_memrotate90_tiled_unpacked(const T *src, int w, int h, int sstrid
 
 template <class T>
 Q_STATIC_TEMPLATE_FUNCTION
-inline void qt_memrotate270_tiled(const T *src, int w, int h, int sstride, T *dest, int dstride)
+inline void qt_memrotate270_tiled(const T *src, int w, int h, int isstride, T *dest, int idstride)
 {
-    sstride /= sizeof(T);
-    dstride /= sizeof(T);
+    const qsizetype sstride = isstride / sizeof(T);
+    const qsizetype dstride = idstride / sizeof(T);
 
     const int pack = sizeof(quint32) / sizeof(T);
     const int unaligned =
@@ -191,9 +192,10 @@ inline void qt_memrotate270_tiled(const T *src, int w, int h, int sstride, T *de
 
 template <class T>
 Q_STATIC_TEMPLATE_FUNCTION
-inline void qt_memrotate270_tiled_unpacked(const T *src, int w, int h, int sstride, T *dest,
-                                           int dstride)
+inline void qt_memrotate270_tiled_unpacked(const T *src, int w, int h, int isstride, T *dest, int idstride)
 {
+    const qsizetype sstride = isstride;
+    const qsizetype dstride = idstride;
     const int numTilesX = (w + tileSize - 1) / tileSize;
     const int numTilesY = (h + tileSize - 1) / tileSize;
 
@@ -247,8 +249,11 @@ inline void qt_memrotate90_template<quint64>(const quint64 *src, int w, int h, i
 
 template <class T>
 Q_STATIC_TEMPLATE_FUNCTION
-inline void qt_memrotate180_template(const T *src, int w, int h, int sstride, T *dest, int dstride)
+inline void qt_memrotate180_template(const T *src, int w, int h, int isstride, T *dest, int idstride)
 {
+    const qsizetype sstride = isstride;
+    const qsizetype dstride = idstride;
+
     const char *s = (const char*)(src) + (h - 1) * sstride;
     for (int dy = 0; dy < h; ++dy) {
         T *d = reinterpret_cast<T*>((char *)(dest) + dy * dstride);