Fix formats conversion in ffmpeg video frames

We convert frames if ffmpeg avPixelFormat is not directly mapped
to qt pixel format.
E.g. av pixel format is AV_PIX_FMT_RGBA64,
we map it qt format RGBA8888, this means that we should convert
AV_PIX_FMT_RGBA64 => AV_PIX_FMT_RGBA to ensure proper data,
this mechanism was broken, we got broken rendering outputs for such
formats.

The patch fixes the conversion.

Task-number: QTBUG-107563
Task-number: QTBUG-108403
Change-Id: I952677bf161195267662c3b170271618ce301edf
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
(cherry picked from commit f0c75fafaa6eeed720a287f081ed5c340fbf64cb)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit ab46759ee7b7efd6232c8e74da584542a7bc8835)

1 files changed, 8 insertions, 7 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index 350a4c8dc..4fdb80cae 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -42,19 +42,20 @@ QFFmpegVideoBuffer::~QFFmpegVideoBuffer() = default;
 void QFFmpegVideoBuffer::convertSWFrame()
 {
     Q_ASSERT(swFrame);
-    bool needsConversion = false;
-    auto pixelFormat = toQtPixelFormat(AVPixelFormat(swFrame->format), &needsConversion);
-    if (pixelFormat != m_pixelFormat || isFrameFlipped(*swFrame)) {
-        AVPixelFormat newFormat = toAVPixelFormat(m_pixelFormat);
+
+    const auto actualAVPixelFormat = AVPixelFormat(swFrame->format);
+    const auto targetAVPixelFormat = toAVPixelFormat(m_pixelFormat);
+    if (actualAVPixelFormat != targetAVPixelFormat || isFrameFlipped(*swFrame)) {
+        Q_ASSERT(toQtPixelFormat(targetAVPixelFormat) == m_pixelFormat);
         // convert the format into something we can handle
-        SwsContext *c = sws_getContext(swFrame->width, swFrame->height, AVPixelFormat(swFrame->format),
-                                       swFrame->width, swFrame->height, newFormat,
+        SwsContext *c = sws_getContext(swFrame->width, swFrame->height, actualAVPixelFormat,
+                                       swFrame->width, swFrame->height, targetAVPixelFormat,
                                        SWS_BICUBIC, nullptr, nullptr, nullptr);
 
         auto newFrame = QFFmpeg::makeAVFrame();
         newFrame->width = swFrame->width;
         newFrame->height = swFrame->height;
-        newFrame->format = newFormat;
+        newFrame->format = targetAVPixelFormat;
         av_frame_get_buffer(newFrame.get(), 0);
 
         sws_scale(c, swFrame->data, swFrame->linesize, 0, swFrame->height, newFrame->data, newFrame->linesize);