AVCaptureDeviceFormat - avoid duplicates (OS X/iOS)

Excluding video range (iOS) is not the right way to avoid "duplicates" - with
other devices there can be also duplicates (formats with the same resolutions),
but completely different pixel formats. Since we do not know what they will be in advance,
we take the media subtype from the initial preset for a capture device and use it
as a filter. Update viewfinder and image encoder settings controls.

Change-Id: If20aea24b19b43574d5c3e9bf2ba85f50fc08916
Reviewed-by: Yoann Lopes <yoann.lopes@theqtcompany.com>

6 files changed, 140 insertions, 93 deletions

diff --git a/src/plugins/avfoundation/camera/avfcamerasession.h b/src/plugins/avfoundation/camera/avfcamerasession.h
index 4f418cb1d..9fd0b1828 100644
--- a/src/plugins/avfoundation/camera/avfcamerasession.h
+++ b/src/plugins/avfoundation/camera/avfcamerasession.h
@@ -80,6 +80,7 @@ public:
 
     void addProbe(AVFMediaVideoProbeControl *probe);
     void removeProbe(AVFMediaVideoProbeControl *probe);
+    FourCharCode defaultCodec();
 
 public Q_SLOTS:
     void setState(QCamera::State state);
@@ -119,6 +120,7 @@ private:
     QSet<AVFMediaVideoProbeControl *> m_videoProbes;
     QMutex m_videoProbesMutex;
 
+    FourCharCode m_defaultCodec;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/avfoundation/camera/avfcamerasession.mm b/src/plugins/avfoundation/camera/avfcamerasession.mm
index 98fbb9865..af30dd312 100644
--- a/src/plugins/avfoundation/camera/avfcamerasession.mm
+++ b/src/plugins/avfoundation/camera/avfcamerasession.mm
@@ -41,6 +41,7 @@
 #include "avfmediavideoprobecontrol.h"
 #include "avfcameraviewfindersettingscontrol.h"
 #include "avfimageencodercontrol.h"
+#include "avfcamerautility.h"
 
 #include <CoreFoundation/CoreFoundation.h>
 #include <Foundation/Foundation.h>
@@ -143,6 +144,7 @@ AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent)
    , m_active(false)
    , m_videoInput(nil)
    , m_audioInput(nil)
+   , m_defaultCodec(0)
 {
     m_captureSession = [[AVCaptureSession alloc] init];
     m_observer = [[AVFCameraSessionObserver alloc] initWithCameraSession:this];
@@ -277,6 +279,8 @@ void AVFCameraSession::setState(QCamera::State newState)
         Q_EMIT readyToConfigureConnections();
         [m_captureSession commitConfiguration];
         [m_captureSession startRunning];
+        m_defaultCodec = 0;
+        defaultCodec();
         applyImageEncoderSettings();
         applyViewfinderSettings();
     }
@@ -407,6 +411,25 @@ void AVFCameraSession::removeProbe(AVFMediaVideoProbeControl *probe)
     m_videoProbesMutex.unlock();
 }
 
+FourCharCode AVFCameraSession::defaultCodec()
+{
+    if (!m_defaultCodec) {
+#if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
+        if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
+            if (AVCaptureDevice *device = videoCaptureDevice()) {
+                AVCaptureDeviceFormat *format = device.activeFormat;
+                if (!format || !format.formatDescription)
+                    return m_defaultCodec;
+                m_defaultCodec = CMVideoFormatDescriptionGetCodecType(format.formatDescription);
+            }
+        }
+#else
+    // TODO: extract media subtype.
+#endif
+    }
+    return m_defaultCodec;
+}
+
 void AVFCameraSession::onCameraFrameFetched(const QVideoFrame &frame)
 {
     m_videoProbesMutex.lock();
diff --git a/src/plugins/avfoundation/camera/avfcamerautility.h b/src/plugins/avfoundation/camera/avfcamerautility.h
index 1c13736e4..03a61460f 100644
--- a/src/plugins/avfoundation/camera/avfcamerautility.h
+++ b/src/plugins/avfoundation/camera/avfcamerautility.h
@@ -97,13 +97,17 @@ AVFRational qt_float_to_rational(qreal par, int limit);
 
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
 
-bool qt_is_video_range_subtype(AVCaptureDeviceFormat *format);
+QVector<AVCaptureDeviceFormat *> qt_unique_device_formats(AVCaptureDevice *captureDevice,
+                                                          FourCharCode preferredFormat);
 QSize qt_device_format_resolution(AVCaptureDeviceFormat *format);
 QSize qt_device_format_high_resolution(AVCaptureDeviceFormat *format);
 QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format);
 QVector<AVFPSRange> qt_device_format_framerates(AVCaptureDeviceFormat *format);
-AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &res);
-AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, Float64 fps);
+AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &res,
+                                                     FourCharCode preferredFormat);
+AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice,
+                                                    FourCharCode preferredFormat,
+                                                    Float64 fps);
 AVFrameRateRange *qt_find_supported_framerate_range(AVCaptureDeviceFormat *format, Float64 fps);
 
 #endif
diff --git a/src/plugins/avfoundation/camera/avfcamerautility.mm b/src/plugins/avfoundation/camera/avfcamerautility.mm
index 554c25cb8..f8d5647eb 100644
--- a/src/plugins/avfoundation/camera/avfcamerautility.mm
+++ b/src/plugins/avfoundation/camera/avfcamerautility.mm
@@ -37,6 +37,7 @@
 #include <QtCore/qvector.h>
 #include <QtCore/qpair.h>
 
+#include <functional>
 #include <algorithm>
 #include <limits>
 
@@ -107,19 +108,6 @@ AVFRational qt_float_to_rational(qreal par, int limit)
 
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
 
-bool qt_is_video_range_subtype(AVCaptureDeviceFormat *format)
-{
-    Q_ASSERT(format);
-#ifdef Q_OS_IOS
-    // Use only 420f on iOS, not 420v.
-    const FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
-    return subType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
-#else
-    Q_UNUSED(format)
-#endif
-    return false;
-}
-
 namespace {
 
 inline bool qt_area_sane(const QSize &size)
@@ -128,40 +116,25 @@ inline bool qt_area_sane(const QSize &size)
            && std::numeric_limits<int>::max() / size.width() >= size.height();
 }
 
-inline bool avf_format_compare(AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2)
+struct ResolutionPredicate : std::binary_function<AVCaptureDeviceFormat *, AVCaptureDeviceFormat *, bool>
 {
-    Q_ASSERT(f1);
-    Q_ASSERT(f2);
-    const QSize r1(qt_device_format_resolution(f1));
-    const QSize r2(qt_device_format_resolution(f2));
-    return r1.width() > r2.width() && r1.height() > r2.height();
-}
+    bool operator() (AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2)const
+    {
+        Q_ASSERT(f1 && f2);
+        const QSize r1(qt_device_format_resolution(f1));
+        const QSize r2(qt_device_format_resolution(f2));
+        return r1.width() < r2.width() || (r2.width() == r1.width() && r1.height() < r2.height());
+    }
+};
 
-QVector<AVCaptureDeviceFormat *> qt_sort_device_formats(AVCaptureDevice *captureDevice)
+struct FormatHasNoFPSRange : std::unary_function<AVCaptureDeviceFormat *, bool>
 {
-    // Select only formats with framerate ranges + sort them by resoluions,
-    Q_ASSERT(captureDevice);
-
-    QVector<AVCaptureDeviceFormat *>sorted;
-
-    NSArray *formats = captureDevice.formats;
-    if (!formats || !formats.count)
-        return sorted;
-
-    sorted.reserve(formats.count);
-    for (AVCaptureDeviceFormat *format in formats) {
-        if (qt_is_video_range_subtype(format))
-            continue;
-        if (format.videoSupportedFrameRateRanges && format.videoSupportedFrameRateRanges.count) {
-            const QSize resolution(qt_device_format_resolution(format));
-            if (!resolution.isNull() && resolution.isValid())
-                sorted << format;
-        }
+    bool operator() (AVCaptureDeviceFormat *format)
+    {
+        Q_ASSERT(format);
+        return !format.videoSupportedFrameRateRanges || !format.videoSupportedFrameRateRanges.count;
     }
-
-    std::sort(sorted.begin(), sorted.end(), avf_format_compare);
-    return sorted;
-}
+};
 
 Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fps)
 {
@@ -180,6 +153,50 @@ Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fp
 
 } // Unnamed namespace.
 
+QVector<AVCaptureDeviceFormat *> qt_unique_device_formats(AVCaptureDevice *captureDevice, FourCharCode filter)
+{
+    // 'filter' is the format we prefer if we have duplicates.
+    Q_ASSERT(captureDevice);
+
+    QVector<AVCaptureDeviceFormat *> formats;
+
+    if (!captureDevice.formats || !captureDevice.formats.count)
+        return formats;
+
+    formats.reserve(captureDevice.formats.count);
+    for (AVCaptureDeviceFormat *format in captureDevice.formats) {
+        const QSize resolution(qt_device_format_resolution(format));
+        if (resolution.isNull() || !resolution.isValid())
+            continue;
+        formats << format;
+    }
+
+    if (!formats.size())
+        return formats;
+
+    std::sort(formats.begin(), formats.end(), ResolutionPredicate());
+
+    QSize size(qt_device_format_resolution(formats[0]));
+    FourCharCode codec = CMVideoFormatDescriptionGetCodecType(formats[0].formatDescription);
+    int last = 0;
+    for (int i = 1; i < formats.size(); ++i) {
+        const QSize nextSize(qt_device_format_resolution(formats[i]));
+        if (nextSize == size) {
+            if (codec == filter)
+                continue;
+            formats[last] = formats[i];
+        } else {
+            ++last;
+            formats[last] = formats[i];
+            size = nextSize;
+        }
+        codec = CMVideoFormatDescriptionGetCodecType(formats[i].formatDescription);
+    }
+    formats.resize(last + 1);
+
+    return formats;
+}
+
 QSize qt_device_format_resolution(AVCaptureDeviceFormat *format)
 {
     Q_ASSERT(format);
@@ -246,7 +263,9 @@ QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format)
     return QSize(asRatio.first, asRatio.second);
 }
 
-AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &request)
+AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice,
+                                                     const QSize &request,
+                                                     FourCharCode filter)
 {
     Q_ASSERT(captureDevice);
     Q_ASSERT(!request.isNull() && request.isValid());
@@ -254,9 +273,10 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
     if (!captureDevice.formats || !captureDevice.formats.count)
         return 0;
 
-    for (AVCaptureDeviceFormat *format in captureDevice.formats) {
-        if (qt_is_video_range_subtype(format))
-            continue;
+    QVector<AVCaptureDeviceFormat *> formats(qt_unique_device_formats(captureDevice, filter));
+
+    for (int i = 0; i < formats.size(); ++i) {
+        AVCaptureDeviceFormat *format = formats[i];
         if (qt_device_format_resolution(format) == request)
             return format;
         // iOS only (still images).
@@ -269,31 +289,30 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
 
     typedef QPair<QSize, AVCaptureDeviceFormat *> FormatPair;
 
-    QVector<FormatPair> formats;
-    formats.reserve(captureDevice.formats.count);
+    QVector<FormatPair> pairs; // default|HR sizes
+    pairs.reserve(formats.size());
 
-    for (AVCaptureDeviceFormat *format in captureDevice.formats) {
-        if (qt_is_video_range_subtype(format))
-            continue;
+    for (int i = 0; i < formats.size(); ++i) {
+        AVCaptureDeviceFormat *format = formats[i];
         const QSize res(qt_device_format_resolution(format));
         if (!res.isNull() && res.isValid() && qt_area_sane(res))
-            formats << FormatPair(res, format);
+            pairs << FormatPair(res, format);
         const QSize highRes(qt_device_format_high_resolution(format));
         if (!highRes.isNull() && highRes.isValid() && qt_area_sane(highRes))
-            formats << FormatPair(highRes, format);
+            pairs << FormatPair(highRes, format);
     }
 
-    if (!formats.size())
+    if (!pairs.size())
         return 0;
 
-    AVCaptureDeviceFormat *best = formats[0].second;
-    QSize next(formats[0].first);
+    AVCaptureDeviceFormat *best = pairs[0].second;
+    QSize next(pairs[0].first);
     int wDiff = qAbs(request.width() - next.width());
     int hDiff = qAbs(request.height() - next.height());
     const int area = request.width() * request.height();
     int areaDiff = qAbs(area - next.width() * next.height());
-    for (int i = 1; i < formats.size(); ++i) {
-        next = formats[i].first;
+    for (int i = 1; i < pairs.size(); ++i) {
+        next = pairs[i].first;
         const int newWDiff = qAbs(next.width() - request.width());
         const int newHDiff = qAbs(next.height() - request.height());
         const int newAreaDiff = qAbs(area - next.width() * next.height());
@@ -302,7 +321,7 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
             || ((newWDiff <= wDiff || newHDiff <= hDiff) && newAreaDiff <= areaDiff)) {
             wDiff = newWDiff;
             hDiff = newHDiff;
-            best = formats[i].second;
+            best = pairs[i].second;
             areaDiff = newAreaDiff;
         }
     }
@@ -310,15 +329,21 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
     return best;
 }
 
-AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, Float64 fps)
+AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice,
+                                                    FourCharCode filter,
+                                                    Float64 fps)
 {
     Q_ASSERT(captureDevice);
     Q_ASSERT(fps > 0.);
 
     const qreal epsilon = 0.1;
 
-    // Sort formats by their resolution.
-    const QVector<AVCaptureDeviceFormat *> sorted(qt_sort_device_formats(captureDevice));
+    QVector<AVCaptureDeviceFormat *>sorted(qt_unique_device_formats(captureDevice, filter));
+    // Sort formats by their resolution in decreasing order:
+    std::sort(sorted.begin(), sorted.end(), std::not2(ResolutionPredicate()));
+    // We can use only formats with framerate ranges:
+    sorted.erase(std::remove_if(sorted.begin(), sorted.end(), FormatHasNoFPSRange()), sorted.end());
+
     if (!sorted.size())
         return nil;
 
diff --git a/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm b/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm
index c5aa5733d..250aae9c1 100644
--- a/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm
+++ b/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm
@@ -279,9 +279,11 @@ QList<QCameraViewfinderSettings> AVFCameraViewfinderSettingsControl2::supportedV
             return supportedSettings;
         }
 
-        for (AVCaptureDeviceFormat *format in m_captureDevice.formats) {
-            if (qt_is_video_range_subtype(format))
-                continue;
+        const QVector<AVCaptureDeviceFormat *> formats(qt_unique_device_formats(m_captureDevice,
+                                                       m_session->defaultCodec()));
+        for (int i = 0; i < formats.size(); ++i) {
+            AVCaptureDeviceFormat *format = formats[i];
+
             const QSize res(qt_device_format_resolution(format));
             if (res.isNull() || !res.isValid())
                 continue;
@@ -435,12 +437,14 @@ AVCaptureDeviceFormat *AVFCameraViewfinderSettingsControl2::findBestFormatMatch(
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
     if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
         Q_ASSERT(m_captureDevice);
+        Q_ASSERT(m_session);
 
         const QSize &resolution = settings.resolution();
         if (!resolution.isNull() && resolution.isValid()) {
             // Either the exact match (including high resolution for images on iOS)
             // or a format with a resolution close to the requested one.
-            return qt_find_best_resolution_match(m_captureDevice, resolution);
+            return qt_find_best_resolution_match(m_captureDevice, resolution,
+                                                 m_session->defaultCodec());
         }
 
         // No resolution requested, what about framerates?
@@ -453,7 +457,8 @@ AVCaptureDeviceFormat *AVFCameraViewfinderSettingsControl2::findBestFormatMatch(
         const qreal minFPS(settings.minimumFrameRate());
         const qreal maxFPS(settings.maximumFrameRate());
         if (minFPS || maxFPS)
-            return qt_find_best_framerate_match(m_captureDevice, maxFPS ? maxFPS : minFPS);
+            return qt_find_best_framerate_match(m_captureDevice, maxFPS ? maxFPS : minFPS,
+                                                m_session->defaultCodec());
         // Ignore PAR for the moment (PAR without resolution can
         // pick a format with really bad resolution).
         // No need to test pixel format, just return settings.
diff --git a/src/plugins/avfoundation/camera/avfimageencodercontrol.mm b/src/plugins/avfoundation/camera/avfimageencodercontrol.mm
index ea25665eb..36050c3a2 100644
--- a/src/plugins/avfoundation/camera/avfimageencodercontrol.mm
+++ b/src/plugins/avfoundation/camera/avfimageencodercontrol.mm
@@ -48,20 +48,6 @@
 
 QT_BEGIN_NAMESPACE
 
-QSize qt_image_high_resolution(AVCaptureDeviceFormat *format)
-{
-    Q_ASSERT(format);
-    QSize res;
-#if defined(Q_OS_IOS) && QT_IOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__IPHONE_8_0)
-    if (QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_8_0) {
-        const CMVideoDimensions hrDim(format.highResolutionStillImageDimensions);
-        res.setWidth(hrDim.width);
-        res.setHeight(hrDim.height);
-    }
-#endif
-    return res;
-}
-
 AVFImageEncoderControl::AVFImageEncoderControl(AVFCameraService *service)
     : m_service(service)
 {
@@ -94,9 +80,11 @@ QList<QSize> AVFImageEncoderControl::supportedResolutions(const QImageEncoderSet
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
     if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
         AVCaptureDevice *captureDevice = m_service->session()->videoCaptureDevice();
-        for (AVCaptureDeviceFormat *format in captureDevice.formats) {
-            if (qt_is_video_range_subtype(format))
-                continue;
+        const QVector<AVCaptureDeviceFormat *> formats(qt_unique_device_formats(captureDevice,
+                                                       m_service->session()->defaultCodec()));
+
+        for (int i = 0; i < formats.size(); ++i) {
+            AVCaptureDeviceFormat *format = formats[i];
 
             const QSize res(qt_device_format_resolution(format));
             if (!res.isNull() && res.isValid())
@@ -108,7 +96,7 @@ QList<QSize> AVFImageEncoderControl::supportedResolutions(const QImageEncoderSet
                 // its source AVCaptureDevice instance’s activeFormat.formatDescription. However,
                 // if you set this property to YES, the receiver emits still images at the capture
                 // device’s highResolutionStillImageDimensions value.
-                const QSize hrRes(qt_image_high_resolution(format));
+                const QSize hrRes(qt_device_format_high_resolution(format));
                 if (!hrRes.isNull() && hrRes.isValid())
                     resolutions << res;
             }
@@ -152,7 +140,7 @@ QImageEncoderSettings AVFImageEncoderControl::imageSettings() const
 
             AVCaptureStillImageOutput *stillImageOutput = m_service->imageCaptureControl()->stillImageOutput();
             if (stillImageOutput.highResolutionStillImageOutputEnabled)
-                res = qt_image_high_resolution(captureDevice.activeFormat);
+                res = qt_device_format_high_resolution(captureDevice.activeFormat);
         }
 #endif
         if (res.isNull() || !res.isValid()) {
@@ -179,7 +167,6 @@ void AVFImageEncoderControl::setImageSettings(const QImageEncoderSettings &setti
         return;
 
     m_settings = settings;
-
     applySettings();
 }
 
@@ -223,7 +210,8 @@ void AVFImageEncoderControl::applySettings()
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
     if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
         AVCaptureDevice *captureDevice = m_service->session()->videoCaptureDevice();
-        AVCaptureDeviceFormat *match = qt_find_best_resolution_match(captureDevice, res);
+        AVCaptureDeviceFormat *match = qt_find_best_resolution_match(captureDevice, res,
+                                                                     m_service->session()->defaultCodec());
 
         if (!match) {
             qDebugCamera() << Q_FUNC_INFO << "unsupported resolution:" << res;
@@ -242,7 +230,7 @@ void AVFImageEncoderControl::applySettings()
 #if defined(Q_OS_IOS) && QT_IOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__IPHONE_8_0)
         if (QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_8_0) {
             AVCaptureStillImageOutput *imageOutput = m_service->imageCaptureControl()->stillImageOutput();
-            if (res == qt_image_high_resolution(captureDevice.activeFormat))
+            if (res == qt_device_format_high_resolution(captureDevice.activeFormat))
                 imageOutput.highResolutionStillImageOutputEnabled = YES;
             else
                 imageOutput.highResolutionStillImageOutputEnabled = NO;