Fix recognizing ambiguous formats by macOS camera implementation

The problem is that we have ambiguous conversions videotoolbox
pixel formats to QVideoFrameFormat::PixelFormat,
e.g. we interpret both
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange and
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
as QVideoFrameFormat::Format_NV12.

Let's store color range in QCameraFormatPrivate to avoid
the ambiguous conversion.

Task-number: QTBUG-115575
Pick-to: 6.6 6.5
Change-Id: I86dc3b85ca111855435acf63fba55ea0f87200bd
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>

2 files changed, 10 insertions, 3 deletions

diff --git a/src/multimedia/platform/qplatformcamera.cpp b/src/multimedia/platform/qplatformcamera.cpp
index 77f1a327a..0d3975550 100644
--- a/src/multimedia/platform/qplatformcamera.cpp
+++ b/src/multimedia/platform/qplatformcamera.cpp
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include "qplatformcamera_p.h"
+#include "private/qcameradevice_p.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -22,13 +23,15 @@ QCameraFormat QPlatformCamera::findBestCameraFormat(const QCameraDevice &camera)
         const auto isValid = fmt.pixelFormat() != QVideoFrameFormat::Format_Invalid;
         const auto resolution = fmt.resolution();
         const auto sufficientFrameRate = std::min(fmt.maxFrameRate(), MinSufficientFrameRate);
+        const auto pixelFormatScore =
+                cameraPixelFormatScore(fmt.pixelFormat(), QCameraFormatPrivate::getColorRange(fmt));
 
         return std::make_tuple(
                 isValid, // 1st: ensure valid formats
                 sufficientFrameRate, // 2nd: ensure the highest frame rate in the range [0; 29]*/
                 resolution.width() * resolution.height(), // 3rd: ensure the highest resolution
-                cameraPixelFormatScore(fmt.pixelFormat()),
-                fmt.maxFrameRate()); // 4th: ensure the highest framerate in the whole range
+                pixelFormatScore, // 4th: eshure the best pixel format
+                fmt.maxFrameRate()); // 5th: ensure the highest framerate in the whole range
     };
 
     const auto formats = camera.videoFormats();
diff --git a/src/multimedia/platform/qplatformcamera_p.h b/src/multimedia/platform/qplatformcamera_p.h
index 794e4372e..85624c0ce 100644
--- a/src/multimedia/platform/qplatformcamera_p.h
+++ b/src/multimedia/platform/qplatformcamera_p.h
@@ -116,7 +116,11 @@ Q_SIGNALS:
 protected:
     explicit QPlatformCamera(QCamera *parent);
 
-    virtual int cameraPixelFormatScore(QVideoFrameFormat::PixelFormat /*format*/) const { return 0; }
+    virtual int cameraPixelFormatScore(QVideoFrameFormat::PixelFormat /*format*/,
+                                       QVideoFrameFormat::ColorRange /*colorRange*/) const
+    {
+        return 0;
+    }
 
     QCameraFormat findBestCameraFormat(const QCameraDevice &camera) const;
     QCameraFormat m_cameraFormat;