Make bytes-per-line safe for int overflow

Goes through the Qt code and make sure bytes-per-line calculations are
safe when they are too big for 32bit integers.

Change-Id: I88b2d74b3da82e91407d316aa932a4a37587c0cf
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

10 files changed, 23 insertions, 20 deletions

diff --git a/src/plugins/imageformats/gif/qgifhandler.cpp b/src/plugins/imageformats/gif/qgifhandler.cpp
index c92cc3ea61..a672e92006 100644
--- a/src/plugins/imageformats/gif/qgifhandler.cpp
+++ b/src/plugins/imageformats/gif/qgifhandler.cpp
@@ -246,7 +246,7 @@ int QGIFFormat::decode(QImage *image, const uchar *buffer, int length,
     }
 
     image->detach();
-    int bpl = image->bytesPerLine();
+    qsizetype bpl = image->bytesPerLine();
     unsigned char *bits = image->bits();
 
 #define LM(l, m) (((m)<<8)|l)
@@ -422,7 +422,7 @@ int QGIFFormat::decode(QImage *image, const uchar *buffer, int length,
                         }
                         memset(backingstore.bits(), 0, backingstore.sizeInBytes());
                     }
-                    const int dest_bpl = backingstore.bytesPerLine();
+                    const qsizetype dest_bpl = backingstore.bytesPerLine();
                     unsigned char *dest_data = backingstore.bits();
                     for (int ln=0; ln<h; ln++) {
                         memcpy(FAST_SCAN_LINE(dest_data, dest_bpl, ln),
diff --git a/src/plugins/imageformats/ico/qicohandler.cpp b/src/plugins/imageformats/ico/qicohandler.cpp
index eb50f52bd6..1982e05344 100644
--- a/src/plugins/imageformats/ico/qicohandler.cpp
+++ b/src/plugins/imageformats/ico/qicohandler.cpp
@@ -356,7 +356,7 @@ void ICOReader::read1BitBMP(QImage & image)
     if (iod) {
 
         int h = image.height();
-        int bpl = image.bytesPerLine();
+        qsizetype bpl = image.bytesPerLine();
 
         while (--h >= 0) {
             if (iod->read((char*)image.scanLine(h),bpl) != bpl) {
@@ -405,7 +405,7 @@ void ICOReader::read8BitBMP(QImage & image)
     if (iod) {
 
         int h = icoAttrib.h;
-        int bpl = image.bytesPerLine();
+        qsizetype bpl = image.bytesPerLine();
 
         while (--h >= 0) {
             if (iod->read((char *)image.scanLine(h), bpl) != bpl) {
@@ -425,7 +425,7 @@ void ICOReader::read16_24_32BMP(QImage & image)
         QRgb *p;
         QRgb  *end;
         uchar *buf = new uchar[image.bytesPerLine()];
-        int    bpl = ((icoAttrib.w*icoAttrib.nbits+31)/32)*4;
+        qsizetype    bpl = ((qsizetype(icoAttrib.w)*icoAttrib.nbits+31)/32)*4;
         uchar *b;
 
         while (--h >= 0) {
diff --git a/src/plugins/platforms/cocoa/qpaintengine_mac.mm b/src/plugins/platforms/cocoa/qpaintengine_mac.mm
index 00b2267f0d..cadb76d2e4 100644
--- a/src/plugins/platforms/cocoa/qpaintengine_mac.mm
+++ b/src/plugins/platforms/cocoa/qpaintengine_mac.mm
@@ -90,7 +90,7 @@ CGImageRef qt_mac_create_imagemask(const QPixmap &pixmap, const QRectF &sr)
         image = image.convertToFormat(QImage::Format_ARGB32_Premultiplied);
 
     const int sx = qRound(sr.x()), sy = qRound(sr.y()), sw = qRound(sr.width()), sh = qRound(sr.height());
-    const int sbpr = image.bytesPerLine();
+    const qsizetype sbpr = image.bytesPerLine();
     const uint nbytes = sw * sh;
     //  alpha is always 255 for bitmaps, ignore it in this case.
     const quint32 mask = pixmap.depth() == 1 ? 0x00ffffff : 0xffffffff;
diff --git a/src/plugins/platforms/vnc/qvnc.cpp b/src/plugins/platforms/vnc/qvnc.cpp
index 8390fa19cd..aea8d26983 100644
--- a/src/plugins/platforms/vnc/qvnc.cpp
+++ b/src/plugins/platforms/vnc/qvnc.cpp
@@ -502,7 +502,7 @@ void QRfbRawEncoder::write()
         const quint32 encoding = htonl(0); // raw encoding
         socket->write((char *)&encoding, sizeof(encoding));
 
-        int linestep = screenImage.bytesPerLine();
+        qsizetype linestep = screenImage.bytesPerLine();
         const uchar *screendata = screenImage.scanLine(rect.y)
                                   + rect.x * screenImage.depth() / 8;
 
diff --git a/src/plugins/platforms/windows/qwindowsmime.cpp b/src/plugins/platforms/windows/qwindowsmime.cpp
index fe9e1fe31f..9bc79a10f9 100644
--- a/src/plugins/platforms/windows/qwindowsmime.cpp
+++ b/src/plugins/platforms/windows/qwindowsmime.cpp
@@ -149,7 +149,10 @@ static bool qt_write_dibv5(QDataStream &s, QImage image)
         return false;
 
     //depth will be always 32
-    int bpl_bmp = image.width()*4;
+    qsizetype bpl_bmp = qsizetype(image.width()) * 4;
+    qsizetype size = bpl_bmp * image.height();
+    if (qsizetype(DWORD(size)) != size)
+        return false;
 
     BMP_BITMAPV5HEADER bi;
     ZeroMemory(&bi, sizeof(bi));
@@ -261,11 +264,11 @@ static bool qt_read_dibv5(QDataStream &s, QImage &image)
     const int blue_shift = calc_shift(blue_mask);
     const int alpha_shift =  alpha_mask ? calc_shift(alpha_mask) : 0u;
 
-    const int  bpl = image.bytesPerLine();
+    const qsizetype  bpl = image.bytesPerLine();
     uchar *data = image.bits();
 
     auto *buf24 = new uchar[bpl];
-    const int bpl24 = ((w * nbits + 31) / 32) * 4;
+    const qsizetype bpl24 = ((qsizetype(w) * nbits + 31) / 32) * 4;
 
     while (--h >= 0) {
         QRgb *p = reinterpret_cast<QRgb *>(data + h * bpl);
diff --git a/src/plugins/platforms/xcb/nativepainting/qpaintengine_x11.cpp b/src/plugins/platforms/xcb/nativepainting/qpaintengine_x11.cpp
index 82b6d60bcd..7bf2b38d7d 100644
--- a/src/plugins/platforms/xcb/nativepainting/qpaintengine_x11.cpp
+++ b/src/plugins/platforms/xcb/nativepainting/qpaintengine_x11.cpp
@@ -2016,7 +2016,7 @@ Q_GUI_EXPORT void qt_x11_drawImage(const QRect &rect, const QPoint &pos, const Q
         || (image_byte_order == LSBFirst && bgr_layout))
     {
         im = image.copy(rect);
-        const int iw = im.bytesPerLine() / 4;
+        const qsizetype iw = im.bytesPerLine() / 4;
         uint *data = (uint *)im.bits();
         for (int i=0; i < h; i++) {
             uint *p = data;
diff --git a/src/plugins/platforms/xcb/nativepainting/qpixmap_x11.cpp b/src/plugins/platforms/xcb/nativepainting/qpixmap_x11.cpp
index f86bedbdcd..467e93e64f 100644
--- a/src/plugins/platforms/xcb/nativepainting/qpixmap_x11.cpp
+++ b/src/plugins/platforms/xcb/nativepainting/qpixmap_x11.cpp
@@ -1747,7 +1747,7 @@ XID QX11PlatformPixmap::createBitmapFromImage(const QImage &image)
     int w = img.width();
     int h = img.height();
     int bpl = (w + 7) / 8;
-    int ibpl = img.bytesPerLine();
+    qsizetype ibpl = img.bytesPerLine();
     if (bpl != ibpl) {
         tmp_bits = new uchar[bpl*h];
         bits = (char *)tmp_bits;
@@ -2017,7 +2017,7 @@ QImage QX11PlatformPixmap::toImage(const QXImageWrapper &xiWrapper, const QRect
         }
     } else if (xi->bits_per_pixel == d) {        // compatible depth
         char *xidata = xi->data;                // copy each scanline
-        int bpl = qMin(int(image.bytesPerLine()),xi->bytes_per_line);
+        qsizetype bpl = qMin(image.bytesPerLine(),xi->bytes_per_line);
         for (int y=0; y<xi->height; y++) {
             memcpy(image.scanLine(y), xidata, bpl);
             xidata += xi->bytes_per_line;
@@ -2038,10 +2038,10 @@ QImage QX11PlatformPixmap::toImage(const QXImageWrapper &xiWrapper, const QRect
         uchar *end;
         uchar  use[256];                        // pixel-in-use table
         uchar  pix[256];                        // pixel translation table
-        int    ncols, bpl;
+        int    ncols;
         memset(use, 0, 256);
         memset(pix, 0, 256);
-        bpl = image.bytesPerLine();
+        qsizetype bpl = image.bytesPerLine();
 
         if (x11_mask) {                         // which pixels are used?
             for (int i = 0; i < xi->height; i++) {
diff --git a/src/plugins/platforms/xcb/qxcbbackingstore.cpp b/src/plugins/platforms/xcb/qxcbbackingstore.cpp
index 8f55bc2e96..7330c3c9a3 100644
--- a/src/plugins/platforms/xcb/qxcbbackingstore.cpp
+++ b/src/plugins/platforms/xcb/qxcbbackingstore.cpp
@@ -537,7 +537,7 @@ void QXcbBackingStoreImage::ensureGC(xcb_drawable_t dst)
 static inline void copy_unswapped(char *dst, int dstBytesPerLine, const QImage &img, const QRect &rect)
 {
     const uchar *srcData = img.constBits();
-    const int srcBytesPerLine = img.bytesPerLine();
+    const qsizetype srcBytesPerLine = img.bytesPerLine();
 
     const int leftOffset = rect.left() * img.depth() >> 3;
     const int bottom = rect.bottom() + 1;
@@ -553,7 +553,7 @@ template <class Pixel>
 static inline void copy_swapped(char *dst, const int dstStride, const QImage &img, const QRect &rect)
 {
     const uchar *srcData = img.constBits();
-    const int srcBytesPerLine = img.bytesPerLine();
+    const qsizetype srcBytesPerLine = img.bytesPerLine();
 
     const int left = rect.left();
     const int width = rect.width();
diff --git a/src/plugins/platforms/xcb/qxcbimage.cpp b/src/plugins/platforms/xcb/qxcbimage.cpp
index b0e610dd51..5b5c37fac4 100644
--- a/src/plugins/platforms/xcb/qxcbimage.cpp
+++ b/src/plugins/platforms/xcb/qxcbimage.cpp
@@ -212,7 +212,7 @@ xcb_pixmap_t qt_xcb_XPixmapFromBitmap(QXcbScreen *screen, const QImage &image)
     }
     const int width = bitmap.width();
     const int height = bitmap.height();
-    const int bytesPerLine = bitmap.bytesPerLine();
+    const qsizetype bytesPerLine = bitmap.bytesPerLine();
     int destLineSize = width / 8;
     if (width % 8)
         ++destLineSize;
diff --git a/src/plugins/styles/mac/qmacstyle_mac.mm b/src/plugins/styles/mac/qmacstyle_mac.mm
index e2599e8c6b..d45ee1a694 100644
--- a/src/plugins/styles/mac/qmacstyle_mac.mm
+++ b/src/plugins/styles/mac/qmacstyle_mac.mm
@@ -2766,12 +2766,12 @@ int QMacStyle::styleHint(StyleHint sh, const QStyleOption *opt, const QWidget *w
             }
 
             const QRgb *sptr = (QRgb*)img.bits(), *srow;
-            const int sbpl = img.bytesPerLine();
+            const qsizetype sbpl = img.bytesPerLine();
             const int w = sbpl/4, h = img.height();
 
             QImage img_mask(img.width(), img.height(), QImage::Format_ARGB32);
             QRgb *dptr = (QRgb*)img_mask.bits(), *drow;
-            const int dbpl = img_mask.bytesPerLine();
+            const qsizetype dbpl = img_mask.bytesPerLine();
 
             for (int y = 0; y < h; ++y) {
                 srow = sptr+((y*sbpl)/4);