Allow QImage with more than 2GByte of image data

Changes internal data-size and pointer calculations
to qssize_t.

Adds new sizeInBytes() accessor to read byte size, and
marks the old one deprecated.

Task-number: QTBUG-50912
Change-Id: Idf0c2010542b0ec1c9abef8afd02d6db07f43e6d
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

6 files changed, 54 insertions, 34 deletions

diff --git a/src/gui/image/qbmphandler.cpp b/src/gui/image/qbmphandler.cpp
index 4350a5c192..c232a84e4f 100644
--- a/src/gui/image/qbmphandler.cpp
+++ b/src/gui/image/qbmphandler.cpp
@@ -49,10 +49,10 @@ QT_BEGIN_NAMESPACE
 
 static void swapPixel01(QImage *image)        // 1-bpp: swap 0 and 1 pixels
 {
-    int i;
+    qssize_t i;
     if (image->depth() == 1 && image->colorCount() == 2) {
         uint *p = (uint *)image->bits();
-        int nbytes = image->byteCount();
+        qssize_t nbytes = static_cast<qssize_t>(image->sizeInBytes());
         for (i=0; i<nbytes/4; i++) {
             *p = ~*p;
             p++;
diff --git a/src/gui/image/qimage.cpp b/src/gui/image/qimage.cpp
index fffd7a1ac2..ccfd928dd6 100644
--- a/src/gui/image/qimage.cpp
+++ b/src/gui/image/qimage.cpp
@@ -127,11 +127,11 @@ QImageData * QImageData::create(const QSize &size, QImage::Format format)
     const int bytes_per_line = ((width * depth + 31) >> 5) << 2; // bytes per scanline (must be multiple of 4)
 
     // sanity check for potential overflows
-    if (INT_MAX/depth < width
+    if (std::numeric_limits<int>::max()/depth < width
         || bytes_per_line <= 0
         || height <= 0
-        || INT_MAX/uint(bytes_per_line) < height
-        || INT_MAX/sizeof(uchar *) < uint(height))
+        || std::numeric_limits<qssize_t>::max()/uint(bytes_per_line) < height
+        || std::numeric_limits<int>::max()/sizeof(uchar *) < uint(height))
         return 0;
 
     QScopedPointer<QImageData> d(new QImageData);
@@ -452,7 +452,7 @@ bool QImageData::checkForAlphaPixels() const
     used. For more information see the
     \l {QImage#Image Formats}{Image Formats} section.
 
-    The format(), bytesPerLine(), and byteCount() functions provide
+    The format(), bytesPerLine(), and sizeInBytes() functions provide
     low-level information about the data stored in the image.
 
     The cacheKey() function returns a number that uniquely
@@ -1448,26 +1448,43 @@ void QImage::setDevicePixelRatio(qreal scaleFactor)
 
 /*!
     \since 4.6
+    \obsolete
     Returns the number of bytes occupied by the image data.
 
-    \sa bytesPerLine(), bits(), {QImage#Image Information}{Image
+    Note this method should never be called on an image larger than 2 gigabytes.
+    Instead use sizeInBytes().
+
+    \sa sizeInBytes(), bytesPerLine(), bits(), {QImage#Image Information}{Image
     Information}
 */
 int QImage::byteCount() const
 {
+    Q_ASSERT(!d || d->nbytes < std::numeric_limits<int>::max());
+    return d ? int(d->nbytes) : 0;
+}
+
+/*!
+    \since 5.10
+    Returns the image data size in bytes.
+
+    \sa byteCount(), bytesPerLine(), bits(), {QImage#Image Information}{Image
+    Information}
+*/
+qssize_t QImage::sizeInBytes() const
+{
     return d ? d->nbytes : 0;
 }
 
 /*!
     Returns the number of bytes per image scanline.
 
-    This is equivalent to byteCount() / height().
+    This is equivalent to sizeInBytes() / height() if height() is non-zero.
 
     \sa scanLine()
 */
 int QImage::bytesPerLine() const
 {
-    return (d && d->height) ? d->nbytes / d->height : 0;
+    return d ? d->bytes_per_line : 0;
 }
 
 
@@ -1594,7 +1611,7 @@ const uchar *QImage::constScanLine(int i) const
     data, thus ensuring that this QImage is the only one using the
     current return value.
 
-    \sa scanLine(), byteCount(), constBits()
+    \sa scanLine(), sizeInBytes(), constBits()
 */
 uchar *QImage::bits()
 {
@@ -4675,12 +4692,12 @@ QImage QImage::transformed(const QTransform &matrix, Qt::TransformationMode mode
         if (dImage.d->colortable.size() < 256) {
             // colors are left in the color table, so pick that one as transparent
             dImage.d->colortable.append(0x0);
-            memset(dImage.bits(), dImage.d->colortable.size() - 1, dImage.byteCount());
+            memset(dImage.bits(), dImage.d->colortable.size() - 1, dImage.d->nbytes);
         } else {
-            memset(dImage.bits(), 0, dImage.byteCount());
+            memset(dImage.bits(), 0, dImage.d->nbytes);
         }
     } else
-        memset(dImage.bits(), 0x00, dImage.byteCount());
+        memset(dImage.bits(), 0x00, dImage.d->nbytes);
 
     if (target_format >= QImage::Format_RGB32) {
         // Prevent QPainter from applying devicePixelRatio corrections
@@ -4785,7 +4802,7 @@ QDebug operator<<(QDebug dbg, const QImage &i)
         if (i.colorCount())
             dbg << ",colorCount=" << i.colorCount();
         dbg << ",devicePixelRatio=" << i.devicePixelRatio()
-            << ",bytesPerLine=" << i.bytesPerLine() << ",byteCount=" << i.byteCount();
+            << ",bytesPerLine=" << i.bytesPerLine() << ",sizeInBytes=" << i.sizeInBytes();
     }
     dbg << ')';
     return dbg;
@@ -4807,7 +4824,7 @@ QDebug operator<<(QDebug dbg, const QImage &i)
 
     Returns the number of bytes occupied by the image data.
 
-    \sa byteCount()
+    \sa sizeInBytes()
  */
 
 /*!
diff --git a/src/gui/image/qimage.h b/src/gui/image/qimage.h
index 225ef3d2e8..7ad44cc33e 100644
--- a/src/gui/image/qimage.h
+++ b/src/gui/image/qimage.h
@@ -214,7 +214,10 @@ public:
     const uchar *bits() const;
     const uchar *constBits() const;
 
-    int byteCount() const;
+#if QT_DEPRECATED_SINCE(5, 10)
+    QT_DEPRECATED int byteCount() const;
+#endif
+    qssize_t sizeInBytes() const;
 
     uchar *scanLine(int);
     const uchar *scanLine(int) const;
diff --git a/src/gui/image/qimage_conversions.cpp b/src/gui/image/qimage_conversions.cpp
index aa7cfe9547..6abaa2887e 100644
--- a/src/gui/image/qimage_conversions.cpp
+++ b/src/gui/image/qimage_conversions.cpp
@@ -823,8 +823,8 @@ static bool convert_indexed8_to_ARGB_PM_inplace(QImageData *data, Qt::ImageConve
 
     const int depth = 32;
 
-    const int dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
-    const int nbytes = dst_bytes_per_line * data->height;
+    const qssize_t dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
+    const qssize_t nbytes = dst_bytes_per_line * data->height;
     uchar *const newData = (uchar *)realloc(data->data, nbytes);
     if (!newData)
         return false;
@@ -877,8 +877,8 @@ static bool convert_indexed8_to_ARGB_inplace(QImageData *data, Qt::ImageConversi
 
     const int depth = 32;
 
-    const int dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
-    const int nbytes = dst_bytes_per_line * data->height;
+    const qssize_t dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
+    const qssize_t nbytes = dst_bytes_per_line * data->height;
     uchar *const newData = (uchar *)realloc(data->data, nbytes);
     if (!newData)
         return false;
@@ -945,8 +945,8 @@ static bool convert_indexed8_to_RGB16_inplace(QImageData *data, Qt::ImageConvers
 
     const int depth = 16;
 
-    const int dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
-    const int nbytes = dst_bytes_per_line * data->height;
+    const qssize_t dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
+    const qssize_t nbytes = dst_bytes_per_line * data->height;
     uchar *const newData = (uchar *)realloc(data->data, nbytes);
     if (!newData)
         return false;
@@ -1002,8 +1002,8 @@ static bool convert_RGB_to_RGB16_inplace(QImageData *data, Qt::ImageConversionFl
 
     const int depth = 16;
 
-    const int dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
-    const int src_bytes_per_line = data->bytes_per_line;
+    const qssize_t dst_bytes_per_line = ((data->width * depth + 31) >> 5) << 2;
+    const qssize_t src_bytes_per_line = data->bytes_per_line;
     quint32 *src_data = (quint32 *) data->data;
     quint16 *dst_data = (quint16 *) data->data;
 
@@ -1257,9 +1257,9 @@ void dither_to_Mono(QImageData *dst, const QImageData *src,
     }
 
     uchar *dst_data = dst->data;
-    int dst_bpl = dst->bytes_per_line;
+    qssize_t dst_bpl = dst->bytes_per_line;
     const uchar *src_data = src->data;
-    int src_bpl = src->bytes_per_line;
+    qssize_t src_bpl = src->bytes_per_line;
 
     switch (dithermode) {
     case Diffuse: {
@@ -1912,8 +1912,8 @@ static void convert_Indexed8_to_Alpha8(QImageData *dest, const QImageData *src,
     if (simpleCase)
         memcpy(dest->data, src->data, src->bytes_per_line * src->height);
     else {
-        int size = src->bytes_per_line * src->height;
-        for (int i = 0; i < size; ++i) {
+        qssize_t size = src->bytes_per_line * src->height;
+        for (qssize_t i = 0; i < size; ++i) {
             dest->data[i] = translate[src->data[i]];
         }
     }
@@ -1936,8 +1936,8 @@ static void convert_Indexed8_to_Grayscale8(QImageData *dest, const QImageData *s
     if (simpleCase)
         memcpy(dest->data, src->data, src->bytes_per_line * src->height);
     else {
-        int size = src->bytes_per_line * src->height;
-        for (int i = 0; i < size; ++i) {
+        qssize_t size = src->bytes_per_line * src->height;
+        for (qssize_t i = 0; i < size; ++i) {
             dest->data[i] = translate[src->data[i]];
         }
     }
diff --git a/src/gui/image/qimage_p.h b/src/gui/image/qimage_p.h
index 775ab6d541..9ba4945dc5 100644
--- a/src/gui/image/qimage_p.h
+++ b/src/gui/image/qimage_p.h
@@ -71,12 +71,12 @@ struct Q_GUI_EXPORT QImageData {        // internal image data
     int width;
     int height;
     int depth;
-    int nbytes;               // number of bytes data
+    qssize_t nbytes;               // number of bytes data
     qreal devicePixelRatio;
     QVector<QRgb> colortable;
     uchar *data;
     QImage::Format format;
-    int bytes_per_line;
+    qssize_t bytes_per_line;
     int ser_no;               // serial number
     int detach_no;
 
diff --git a/src/gui/image/qpixmap_blitter.cpp b/src/gui/image/qpixmap_blitter.cpp
index 950695a9d7..de32327071 100644
--- a/src/gui/image/qpixmap_blitter.cpp
+++ b/src/gui/image/qpixmap_blitter.cpp
@@ -190,8 +190,8 @@ void QBlittablePlatformPixmap::fromImage(const QImage &image,
 
     uchar *mem = thisImg->bits();
     const uchar *bits = correctFormatPic.constBits();
-    int bytesCopied = 0;
-    while (bytesCopied < correctFormatPic.byteCount()) {
+    qssize_t bytesCopied = 0;
+    while (bytesCopied < correctFormatPic.sizeInBytes()) {
         memcpy(mem,bits,correctFormatPic.bytesPerLine());
         mem += thisImg->bytesPerLine();
         bits += correctFormatPic.bytesPerLine();