Fix recognizing ambiguous formats by macOS camera implementation

The problem is that we have ambiguous conversions videotoolbox
pixel formats to QVideoFrameFormat::PixelFormat,
e.g. we interpret both
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange and
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
as QVideoFrameFormat::Format_NV12.

Let's store color range in QCameraFormatPrivate to avoid
the ambiguous conversion.

Task-number: QTBUG-115575
Change-Id: I86dc3b85ca111855435acf63fba55ea0f87200bd
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
(cherry picked from commit 35f59535360e9822150f6daf4d0c95e2efe17176)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

1 files changed, 4 insertions, 5 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera.mm b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
index 0a22e81bd..9b87a5423 100644
--- a/src/plugins/multimedia/ffmpeg/qavfcamera.mm
+++ b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
@@ -314,12 +314,11 @@ std::optional<int> QAVFCamera::ffmpegHWPixelFormat() const
     return m_hwPixelFormat == AV_PIX_FMT_NONE ? std::optional<int>{} : m_hwPixelFormat;
 }
 
-int QAVFCamera::cameraPixelFormatScore(QVideoFrameFormat::PixelFormat pixFmt) const
+int QAVFCamera::cameraPixelFormatScore(QVideoFrameFormat::PixelFormat pixelFormat,
+                                       QVideoFrameFormat::ColorRange colorRange) const
 {
-    unsigned cvFormat = 0;
-    const auto isSupported =
-            QAVFHelpers::toCVPixelFormat(pixFmt, cvFormat) && isCVFormatSupported(cvFormat);
-    return static_cast<int>(isSupported);
+    auto cvFormat = QAVFHelpers::toCVPixelFormat(pixelFormat, colorRange);
+    return static_cast<int>(isCVFormatSupported(cvFormat));
 }
 
 QT_END_NAMESPACE