QImage: merge the size calculations with proper (non-UB) checks

This check, which was only done once, was wrong:
    const int bytes_per_line = ((width * depth + 31) >> 5) << 2;

    // sanity check for potential overflows
    if (std::numeric_limits<int>::max()/depth < width

If width*height overflows, then it's already UB and checking afterwards
with a division is pointless and slow.

The other instances weren't properly guarding against overflows.

Change-Id: I343f2beed55440a7ac0bfffd1563350d4cfa639c
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>

1 files changed, 33 insertions, 0 deletions

diff --git a/src/gui/image/qimage_p.h b/src/gui/image/qimage_p.h
index 2fe29a88d3..e3a6c53833 100644
--- a/src/gui/image/qimage_p.h
+++ b/src/gui/image/qimage_p.h
@@ -52,6 +52,7 @@
 //
 
 #include <QtGui/private/qtguiglobal_p.h>
+#include <QtCore/private/qnumeric_p.h>
 
 #include <QMap>
 #include <QVector>
@@ -104,8 +105,40 @@ struct Q_GUI_EXPORT QImageData {        // internal image data
     bool doImageIO(const QImage *image, QImageWriter* io, int quality) const;
 
     QPaintEngine *paintEngine;
+
+    struct ImageSizeParameters {
+        qsizetype bytesPerLine;
+        qsizetype totalSize;
+    };
+    static ImageSizeParameters calculateImageParameters(qsizetype width, qsizetype height, qsizetype depth);
 };
 
+inline QImageData::ImageSizeParameters
+QImageData::calculateImageParameters(qsizetype width, qsizetype height, qsizetype depth)
+{
+    ImageSizeParameters invalid = { -1, -1 };
+    if (height <= 0)
+        return invalid;
+
+    // calculate the size, taking care of overflows
+    qsizetype bytes_per_line;
+    if (mul_overflow(width, depth, &bytes_per_line))
+        return invalid;
+    if (add_overflow(bytes_per_line, qsizetype(31), &bytes_per_line))
+        return invalid;
+    // bytes per scanline (must be multiple of 4)
+    bytes_per_line = (bytes_per_line >> 5) << 2;    // can't overflow
+
+    qsizetype total_size;
+    if (mul_overflow(height, bytes_per_line, &total_size))
+        return invalid;
+    qsizetype dummy;
+    if (mul_overflow(height, qsizetype(sizeof(uchar *)), &dummy))
+        return invalid;                                 // why is this here?
+
+    return { bytes_per_line, total_size };
+}
+
 typedef void (*Image_Converter)(QImageData *dest, const QImageData *src, Qt::ImageConversionFlags);
 typedef bool (*InPlace_Image_Converter)(QImageData *data, Qt::ImageConversionFlags);