Ensure valid data after QImage::invertPixels

QImage::invertPixels may produce invalid data after inversions of
images with premultiplied alpha, because the inverted colors will be
larger than the alpha.

This patch converts any image with a premultiplied alpha channel to
ARGB32 before inverting the pixels, and then back to the original
format after the inversion.

Support is added for correct inversion of RGBA8888 and RGB30 formats.

Task-number: QTBUG-39901
Change-Id: Ief24c55f495e67ef2ad6429b5b418d02963a64dd
Reviewed-by: Gunnar Sletta <gunnar.sletta@jollamobile.com>

2 files changed, 85 insertions, 3 deletions

diff --git a/src/gui/image/qimage.cpp b/src/gui/image/qimage.cpp
index 8c7c446e96..0d10dd77d2 100644
--- a/src/gui/image/qimage.cpp
+++ b/src/gui/image/qimage.cpp
@@ -1806,6 +1806,9 @@ void QImage::fill(const QColor &color)
     is the case for a 1-bit image. Note that the color table is \e not
     changed.
 
+    If the image has a premultiplied alpha channel, the image is first
+    converted to ARGB32 to be inverted and then converted back.
+
     \sa {QImage#Image Transformations}{Image Transformations}
 */
 
@@ -1820,8 +1823,15 @@ void QImage::invertPixels(InvertMode mode)
     if (!d)
         return;
 
-    if (depth() != 32) {
-        // number of used bytes pr line
+    QImage::Format originalFormat = d->format;
+    // Inverting premultiplied pixels would produce invalid image data.
+    if (hasAlphaChannel() && qPixelLayouts[d->format].premultiplied) {
+        if (!d->convertInPlace(QImage::Format_ARGB32, 0))
+            *this = convertToFormat(QImage::Format_ARGB32);
+    }
+
+    if (depth() < 32) {
+        // This assumes no alpha-channel as the only formats with non-premultipled alpha are 32bit.
         int bpl = (d->width * d->depth + 7) / 8;
         int pad = d->bytes_per_line - bpl;
         uchar *sl = d->data;
@@ -1833,10 +1843,44 @@ void QImage::invertPixels(InvertMode mode)
     } else {
         quint32 *p = (quint32*)d->data;
         quint32 *end = (quint32*)(d->data + d->nbytes);
-        uint xorbits = (mode == InvertRgba) ? 0xffffffff : 0x00ffffff;
+        quint32 xorbits = 0xffffffff;
+        switch (d->format) {
+        case QImage::Format_RGBA8888:
+            if (mode == InvertRgba)
+                break;
+            // no break
+        case QImage::Format_RGBX8888:
+#if Q_BYTE_ORDER == Q_BIG_ENDIAN
+            xorbits = 0xffffff00;
+            break;
+#else
+            xorbits = 0x00ffffff;
+            break;
+#endif
+        case QImage::Format_ARGB32:
+            if (mode == InvertRgba)
+                break;
+            // no break
+        case QImage::Format_RGB32:
+            xorbits = 0x00ffffff;
+            break;
+        case QImage::Format_BGR30:
+        case QImage::Format_RGB30:
+            xorbits = 0x3fffffff;
+            break;
+        default:
+            Q_UNREACHABLE();
+            xorbits = 0;
+            break;
+        }
         while (p < end)
             *p++ ^= xorbits;
     }
+
+    if (originalFormat != d->format) {
+        if (!d->convertInPlace(originalFormat, 0))
+            *this = convertToFormat(originalFormat);
+    }
 }
 
 // Windows defines these
diff --git a/tests/auto/gui/image/qimage/tst_qimage.cpp b/tests/auto/gui/image/qimage/tst_qimage.cpp
index d885ab6dd4..f922deb815 100644
--- a/tests/auto/gui/image/qimage/tst_qimage.cpp
+++ b/tests/auto/gui/image/qimage/tst_qimage.cpp
@@ -178,6 +178,9 @@ private slots:
     void convertToImageFormat_data();
     void convertToImageFormat();
 
+    void invertPixelsRGB_data();
+    void invertPixelsRGB();
+
     void cleanupFunctions();
 
 private:
@@ -2585,6 +2588,7 @@ void tst_QImage::convertToImageFormat_data()
     QTest::newRow("Convert Format_RGBA8888") << QImage::Format_RGBA8888;
     QTest::newRow("Convert Format_RGBA8888_Premultiplied") << QImage::Format_RGBA8888_Premultiplied;
 }
+
 void tst_QImage::convertToImageFormat()
 {
     QFETCH(QImage::Format, image_format);
@@ -2594,6 +2598,40 @@ void tst_QImage::convertToImageFormat()
     QCOMPARE(format, image_format);
 }
 
+void tst_QImage::invertPixelsRGB_data()
+{
+    QTest::addColumn<QImage::Format>("image_format");
+
+    QTest::newRow("invertPixels RGB16") << QImage::Format_RGB16;
+    QTest::newRow("invertPixels RGB32") << QImage::Format_RGB32;
+    QTest::newRow("invertPixels BGR30") << QImage::Format_BGR30;
+    QTest::newRow("invertPixels RGB444") << QImage::Format_RGB444;
+    QTest::newRow("invertPixels RGB555") << QImage::Format_RGB555;
+    QTest::newRow("invertPixels RGB888") << QImage::Format_RGB888;
+
+    QTest::newRow("invertPixels ARGB32") << QImage::Format_ARGB32;
+    QTest::newRow("invertPixels ARGB32pm") << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("invertPixels RGBA8888") << QImage::Format_RGBA8888;
+    QTest::newRow("invertPixels RGBA8888pm") << QImage::Format_RGBA8888_Premultiplied;
+    QTest::newRow("invertPixels RGBA4444pm") << QImage::Format_ARGB4444_Premultiplied;
+}
+
+void tst_QImage::invertPixelsRGB()
+{
+    QFETCH(QImage::Format, image_format);
+
+    QImage image(1, 1, image_format);
+    image.fill(QColor::fromRgb(32, 64, 96));
+    image.invertPixels();
+
+    QCOMPARE(image.format(), image_format);
+
+    uint pixel = image.pixel(0, 0);
+    QCOMPARE(qRed(pixel) >> 4, (255 - 32) >> 4);
+    QCOMPARE(qGreen(pixel) >> 4, (255 - 64) >> 4);
+    QCOMPARE(qBlue(pixel) >> 4, (255 - 96) >> 4);
+}
+
 static void cleanupFunction(void* info)
 {
     bool *called = static_cast<bool*>(info);