Implement fixing of QImage by QImageVideoBuffer

Many of QImage formats are not mapped to QVideoFrame formats without
conversions. Let's consider this and ensure
a correct video frame format.

Eglfs screen capture is to be fixed in the next commits.

The relevant unit test is added to the patch.

Pick-to: 6.6 6.5
Change-Id: Ic4a490392ec2b6aa63e752badea7269a07d93af9
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit f9edbcb7fedcd7e2e2c274dcda1979e304245b32)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

4 files changed, 130 insertions, 3 deletions

diff --git a/src/multimedia/video/qimagevideobuffer.cpp b/src/multimedia/video/qimagevideobuffer.cpp
index 293d17252..bc825004e 100644
--- a/src/multimedia/video/qimagevideobuffer.cpp
+++ b/src/multimedia/video/qimagevideobuffer.cpp
@@ -2,11 +2,57 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include "qimagevideobuffer_p.h"
+#include "qvideoframeformat.h"
 
 QT_BEGIN_NAMESPACE
 
+namespace {
+
+QImage::Format fixImageFormat(QImage::Format format)
+{
+    switch (format) {
+    case QImage::Format_ARGB32_Premultiplied:
+    case QImage::Format_ARGB8565_Premultiplied:
+    case QImage::Format_ARGB6666_Premultiplied:
+    case QImage::Format_ARGB8555_Premultiplied:
+    case QImage::Format_ARGB4444_Premultiplied:
+    case QImage::Format_RGBA8888_Premultiplied:
+    case QImage::Format_A2BGR30_Premultiplied:
+    case QImage::Format_A2RGB30_Premultiplied:
+    case QImage::Format_RGBA64_Premultiplied:
+    case QImage::Format_RGBA16FPx4_Premultiplied:
+    case QImage::Format_RGBA32FPx4_Premultiplied:
+        return QImage::Format_ARGB32_Premultiplied;
+    case QImage::Format_ARGB32:
+    case QImage::Format_RGBA8888:
+    case QImage::Format_Alpha8:
+    case QImage::Format_RGBA64:
+    case QImage::Format_RGBA16FPx4:
+    case QImage::Format_RGBA32FPx4:
+        return QImage::Format_ARGB32;
+    case QImage::Format_Invalid:
+        return QImage::Format_Invalid;
+    default:
+        return QImage::Format_RGB32;
+    }
+}
+
+QImage fixImage(QImage image)
+{
+    if (image.format() == QImage::Format_Invalid)
+        return image;
+
+    const auto frameFormat = QVideoFrameFormat::pixelFormatFromImageFormat(image.format());
+    if (frameFormat != QVideoFrameFormat::Format_Invalid)
+        return image;
+
+    return image.convertToFormat(fixImageFormat(image.format()));
+}
+
+} // namespace
+
 QImageVideoBuffer::QImageVideoBuffer(QImage image)
-    : QAbstractVideoBuffer(QVideoFrame::NoHandle), m_image(std::move(image))
+    : QAbstractVideoBuffer(QVideoFrame::NoHandle), m_image(fixImage(std::move(image)))
 {
 }
 
@@ -36,4 +82,9 @@ void QImageVideoBuffer::unmap()
     m_mapMode = QVideoFrame::NotMapped;
 }
 
+QImage QImageVideoBuffer::underlyingImage() const
+{
+    return m_image;
+}
+
 QT_END_NAMESPACE
diff --git a/src/multimedia/video/qimagevideobuffer_p.h b/src/multimedia/video/qimagevideobuffer_p.h
index 83fc21d4f..e5467563a 100644
--- a/src/multimedia/video/qimagevideobuffer_p.h
+++ b/src/multimedia/video/qimagevideobuffer_p.h
@@ -31,6 +31,8 @@ public:
 
     void unmap() override;
 
+    QImage underlyingImage() const;
+
 private:
     QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;
     QImage m_image;
diff --git a/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp b/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp
index 8ed3aecd9..4e87a0559 100644
--- a/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp
+++ b/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp
@@ -125,7 +125,8 @@ private:
         setFrameRate(screen->refreshRate());
 
         QPixmap p = screen->grabWindow(wid);
-        QImage img = p.toImage();
+        auto buffer = std::make_unique<QImageVideoBuffer>(p.toImage());
+        const auto img = buffer->underlyingImage();
 
         QVideoFrameFormat format(img.size(),
                                  QVideoFrameFormat::pixelFormatFromImageFormat(img.format()));
@@ -138,7 +139,7 @@ private:
             return {};
         }
 
-        return QVideoFrame(new QImageVideoBuffer(std::move(img)), format);
+        return QVideoFrame(buffer.release(), format);
     }
 
 private:
diff --git a/tests/auto/unit/multimedia/qvideobuffers/tst_qvideobuffers.cpp b/tests/auto/unit/multimedia/qvideobuffers/tst_qvideobuffers.cpp
index cbbfea91e..0d209fbda 100644
--- a/tests/auto/unit/multimedia/qvideobuffers/tst_qvideobuffers.cpp
+++ b/tests/auto/unit/multimedia/qvideobuffers/tst_qvideobuffers.cpp
@@ -5,6 +5,7 @@
 
 #include <private/qmemoryvideobuffer_p.h>
 #include <private/qimagevideobuffer_p.h>
+#include "qvideoframeformat.h"
 
 using BufferPtr = std::shared_ptr<QAbstractVideoBuffer>;
 using MapModes = std::vector<QVideoFrame::MapMode>;
@@ -40,6 +41,9 @@ private slots:
     void unmap_resetsMappedState_whenBufferIsMapped_data();
     void unmap_resetsMappedState_whenBufferIsMapped();
 
+    void imageBuffer_fixesInputImage_data();
+    void imageBuffer_fixesInputImage();
+
 private:
     QString mapModeToString(QVideoFrame::MapMode mapMode) const
     {
@@ -218,6 +222,75 @@ void tst_QVideoBuffers::unmap_resetsMappedState_whenBufferIsMapped()
     QCOMPARE(QByteArray(data, mappedData.size[0]), m_byteArray);
 }
 
+void tst_QVideoBuffers::imageBuffer_fixesInputImage_data()
+{
+    QTest::addColumn<QImage::Format>("inputImageFormat");
+    QTest::addColumn<QImage::Format>("underlyingImageFormat");
+
+    QTest::newRow("Format_RGB32 => Format_RGB32") << QImage::Format_RGB32 << QImage::Format_RGB32;
+    QTest::newRow("Format_ARGB32 => Format_ARGB32")
+            << QImage::Format_ARGB32 << QImage::Format_ARGB32;
+    QTest::newRow("Format_ARGB32_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_ARGB32_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_RGBA8888 => Format_RGBA8888")
+            << QImage::Format_RGBA8888 << QImage::Format_RGBA8888;
+    QTest::newRow("Format_RGBA8888_Premultiplied => Format_RGBA8888_Premultiplied")
+            << QImage::Format_RGBA8888_Premultiplied << QImage::Format_RGBA8888_Premultiplied;
+    QTest::newRow("Format_RGBX8888 => Format_RGBX8888")
+            << QImage::Format_RGBX8888 << QImage::Format_RGBX8888;
+    QTest::newRow("Format_Grayscale8 => Format_Grayscale8")
+            << QImage::Format_Grayscale8 << QImage::Format_Grayscale8;
+    QTest::newRow("Format_Grayscale16 => Format_Grayscale16")
+            << QImage::Format_Grayscale16 << QImage::Format_Grayscale16;
+
+    QTest::newRow("Format_ARGB8565_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_ARGB8565_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_ARGB6666_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_ARGB6666_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_ARGB8555_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_ARGB8555_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_ARGB4444_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_ARGB4444_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_A2BGR30_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_A2BGR30_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_A2RGB30_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_A2RGB30_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_RGBA64_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_RGBA64_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_RGBA16FPx4_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_RGBA16FPx4_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+    QTest::newRow("Format_RGBA32FPx4_Premultiplied => Format_ARGB32_Premultiplied")
+            << QImage::Format_RGBA32FPx4_Premultiplied << QImage::Format_ARGB32_Premultiplied;
+
+    QTest::newRow("Format_Alpha8 => Format_ARGB32")
+            << QImage::Format_Alpha8 << QImage::Format_ARGB32;
+    QTest::newRow("Format_RGBA64 => Format_ARGB32")
+            << QImage::Format_RGBA64 << QImage::Format_ARGB32;
+    QTest::newRow("Format_RGBA16FPx4 => Format_ARGB32")
+            << QImage::Format_RGBA16FPx4 << QImage::Format_ARGB32;
+    QTest::newRow("Format_RGBA32FPx4 => Format_ARGB32")
+            << QImage::Format_RGBA32FPx4 << QImage::Format_ARGB32;
+}
+
+void tst_QVideoBuffers::imageBuffer_fixesInputImage()
+{
+    QFETCH(QImage::Format, inputImageFormat);
+    QFETCH(QImage::Format, underlyingImageFormat);
+
+    m_image.convertTo(inputImageFormat);
+    QImageVideoBuffer buffer(m_image);
+
+    auto underlyingImage = buffer.underlyingImage();
+
+    QCOMPARE(underlyingImage.format(), underlyingImageFormat);
+    QCOMPARE_NE(QVideoFrameFormat::pixelFormatFromImageFormat(underlyingImage.format()),
+                QVideoFrameFormat::Format_Invalid);
+    QCOMPARE(m_image.convertedTo(underlyingImageFormat), underlyingImage);
+
+    if (inputImageFormat == underlyingImageFormat)
+        QCOMPARE(m_image.constBits(), underlyingImage.constBits());
+}
+
 QTEST_APPLESS_MAIN(tst_QVideoBuffers);
 
 #include "tst_qvideobuffers.moc"