Improve ffmpeg camera formats selection on darwin OS

Handled a bunch of corner cases, briefly they are:
- handle the case when ffmpeg videotoolbox doesn't support format.
  From now, it's optionally available to use sw frame or choose
  another format. Both approaches are working, but for now,
  the first one is prefferable.
- handle the rare case when device output formats don't contain
  camera format.
- improve the best camera format selection, it'll take into
  account an opoortunity to use hw acceleration for the format.
- improve logging.

Task-number: QTBUG-109009
Pick-to: 6.5
Change-Id: I82773f73a47bc1d22730f1e5ec2eaacbfa4ec1b1
Reviewed-by: Lars Knoll <lars@knoll.priv.no>

2 files changed, 45 insertions, 16 deletions

diff --git a/src/multimedia/platform/qplatformcamera.cpp b/src/multimedia/platform/qplatformcamera.cpp
index 9445dff33..74a114ca9 100644
--- a/src/multimedia/platform/qplatformcamera.cpp
+++ b/src/multimedia/platform/qplatformcamera.cpp
@@ -12,23 +12,45 @@ QPlatformCamera::QPlatformCamera(QCamera *parent)
     qRegisterMetaType<QVideoFrame>();
 }
 
-QCameraFormat QPlatformCamera::findBestCameraFormat(const QCameraDevice &camera)
+QCameraFormat QPlatformCamera::findBestCameraFormat(const QCameraDevice &camera) const
 {
-    QCameraFormat f;
+    // check if fmt is better. We try to find the highest resolution that offers
+    // at least 30 FPS
+    // we use 29 FPS to compare against as some cameras report 29.97 FPS...
+
+    auto makeCriteria = [this](const QCameraFormat &fmt) {
+        constexpr float MinSufficientFrameRate = 29.f;
+
+        const auto isValid = fmt.pixelFormat() != QVideoFrameFormat::Format_Invalid;
+        const auto resolution = fmt.resolution();
+        const auto sufficientFrameRate = std::min(fmt.maxFrameRate(), MinSufficientFrameRate);
+
+        return std::make_tuple(
+                isValid, // 1st: ensure valid formats
+                sufficientFrameRate, // 2nd: ensure the highest frame rate in the range [0; 29]*/
+                resolution.width() * resolution.height(), // 3rd: ensure the highest resolution
+                cameraPixelFormatScore(fmt.pixelFormat()),
+                fmt.maxFrameRate()); // 4th: ensure the highest framerate in the whole range
+    };
+
     const auto formats = camera.videoFormats();
-    for (const auto &fmt : formats) {
-        if (fmt.pixelFormat() == QVideoFrameFormat::Format_Invalid)
-            continue;
-        // check if fmt is better. We try to find the highest resolution that offers
-        // at least 30 FPS
-        // we use 29 FPS to compare against as some cameras report 29.97 FPS...
-        if (f.maxFrameRate() < 29 && fmt.maxFrameRate() > f.maxFrameRate())
-            f = fmt;
-        else if (f.maxFrameRate() == fmt.maxFrameRate() &&
-                 f.resolution().width()*f.resolution().height() < fmt.resolution().width()*fmt.resolution().height())
-            f = fmt;
-    }
-    return f;
+    const auto found =
+            std::max_element(formats.begin(), formats.end(),
+                             [makeCriteria](const QCameraFormat &fmtA, const QCameraFormat &fmtB) {
+                                 return makeCriteria(fmtA) < makeCriteria(fmtB);
+                             });
+
+    return found == formats.end() ? QCameraFormat{} : *found;
+}
+
+QVideoFrameFormat QPlatformCamera::frameFormat() const
+{
+    QVideoFrameFormat result(m_cameraFormat.resolution(),
+                             m_framePixelFormat == QVideoFrameFormat::Format_Invalid
+                                     ? m_cameraFormat.pixelFormat()
+                                     : m_framePixelFormat);
+    result.setFrameRate(m_cameraFormat.maxFrameRate());
+    return result;
 }
 
 void QPlatformCamera::supportedFeaturesChanged(QCamera::Features f)
diff --git a/src/multimedia/platform/qplatformcamera_p.h b/src/multimedia/platform/qplatformcamera_p.h
index 62f2836b8..555adeef5 100644
--- a/src/multimedia/platform/qplatformcamera_p.h
+++ b/src/multimedia/platform/qplatformcamera_p.h
@@ -19,6 +19,7 @@
 #include <QtMultimedia/qtmultimediaglobal.h>
 
 #include <QtMultimedia/qcamera.h>
+#include <QtMultimedia/qvideoframeformat.h>
 #include <QtCore/private/qglobal_p.h>
 
 QT_BEGIN_NAMESPACE
@@ -66,6 +67,8 @@ public:
     virtual void setWhiteBalanceMode(QCamera::WhiteBalanceMode /*mode*/) {}
     virtual void setColorTemperature(int /*temperature*/) {}
 
+    QVideoFrameFormat frameFormat() const;
+
     QCamera::Features supportedFeatures() const { return m_supportedFeatures; }
 
     QCamera::FocusMode focusMode() const { return m_focusMode; }
@@ -126,8 +129,12 @@ Q_SIGNALS:
 protected:
     explicit QPlatformCamera(QCamera *parent);
 
-    static QCameraFormat findBestCameraFormat(const QCameraDevice &camera);
+    virtual int cameraPixelFormatScore(QVideoFrameFormat::PixelFormat /*format*/) const { return 0; }
+
+    QCameraFormat findBestCameraFormat(const QCameraDevice &camera) const;
     QCameraFormat m_cameraFormat;
+    QVideoFrameFormat::PixelFormat m_framePixelFormat = QVideoFrameFormat::Format_Invalid;
+
 private:
     QCamera *m_camera = nullptr;
     QCamera::Features m_supportedFeatures = {};