Use the new AVFVideoSink abstraction for the camera as well

This makes the camera work in windowed and not SW rendering modes
again.

Change-Id: I2f8c2c3f445705c3f9a4619ad33330500cf2a487
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
Reviewed-by: Doris Verria <doris.verria@qt.io>

8 files changed, 56 insertions, 29 deletions

diff --git a/src/multimedia/platform/darwin/avfvideosink.mm b/src/multimedia/platform/darwin/avfvideosink.mm
index c7473c70a..1d7147b1f 100644
--- a/src/multimedia/platform/darwin/avfvideosink.mm
+++ b/src/multimedia/platform/darwin/avfvideosink.mm
@@ -131,7 +131,7 @@ void AVFVideoSink::setNativeSize(QSize size)
         return;
     m_nativeSize = size;
     if (m_interface)
-        m_interface->updateNativeSize();
+        m_interface->nativeSizeChanged();
 }
 
 Qt::AspectRatioMode AVFVideoSink::aspectRatioMode() const
@@ -217,12 +217,17 @@ void AVFVideoSinkInterface::setVideoSink(AVFVideoSink *sink)
 
 void AVFVideoSinkInterface::setLayer(CALayer *layer)
 {
+    if (layer == m_layer)
+        return;
+
     if (m_layer) {
         renderToNativeView(false);
         [m_layer release];
     }
     m_layer = layer;
     [m_layer retain];
+
+    reconfigure();
 }
 
 void AVFVideoSinkInterface::renderToNativeView(bool enable)
diff --git a/src/multimedia/platform/darwin/avfvideosink_p.h b/src/multimedia/platform/darwin/avfvideosink_p.h
index c5b953aca..19cdb0b91 100644
--- a/src/multimedia/platform/darwin/avfvideosink_p.h
+++ b/src/multimedia/platform/darwin/avfvideosink_p.h
@@ -140,7 +140,8 @@ public:
 
     virtual void reconfigure() = 0;
     virtual void updateAspectRatio() = 0;
-    virtual void setLayer(CALayer *layer);
+
+    void setLayer(CALayer *layer);
 
     void renderToNativeView(bool enable);
 
@@ -151,7 +152,7 @@ public:
     bool rendersToWindow() const { return m_rendersToWindow; }
 
     void updateLayerBounds();
-    void updateNativeSize() { updateLayerBounds(); }
+    void nativeSizeChanged() { updateLayerBounds(); }
 
 protected:
     NativeView *nativeView() const { return m_sink->nativeView(); }
@@ -163,7 +164,6 @@ protected:
 
     AVFVideoSink *m_sink = nullptr;
     CALayer *m_layer = nullptr;
-    QSize m_nativeSize;
     bool m_rendersToWindow = false;
 };
 
diff --git a/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol.mm b/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol.mm
index 9826ed375..5be8106bd 100644
--- a/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol.mm
+++ b/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol.mm
@@ -284,12 +284,19 @@ AVFCameraRendererControl::~AVFCameraRendererControl()
 #endif
 }
 
-void AVFCameraRendererControl::setVideoSink(AVFVideoSink *sink)
+void AVFCameraRendererControl::reconfigure()
 {
-    if (m_sink == sink)
-        return;
+    QMutexLocker lock(&m_vfMutex);
+
+    renderToNativeView(shouldRenderToWindow());
 
-    m_sink = sink;
+    m_rendersToWindow = rendersToWindow();
+
+    updateAspectRatio();
+    // ### This is a hack, need to use a reliable way to determine the size and not use the preview layer
+    if (m_layer)
+        m_sink->setNativeSize(QSize(m_layer.bounds.size.width, m_layer.bounds.size.height));
+    nativeSizeChanged();
 }
 
 void AVFCameraRendererControl::configureAVCaptureSession(AVFCameraSession *cameraSession)
@@ -331,6 +338,9 @@ void AVFCameraRendererControl::updateCaptureConnection()
 void AVFCameraRendererControl::syncHandleViewfinderFrame(const QVideoFrame &frame)
 {
     QMutexLocker lock(&m_vfMutex);
+    if (m_rendersToWindow)
+        return;
+
     if (!m_lastViewfinderFrame.isValid()) {
         static QMetaMethod handleViewfinderFrameSlot = metaObject()->method(
                     metaObject()->indexOfMethod("handleViewfinderFrame()"));
@@ -376,4 +386,28 @@ void AVFCameraRendererControl::handleViewfinderFrame()
 }
 
 
+void AVFCameraRendererControl::updateAspectRatio()
+{
+    if (!m_layer)
+        return;
+
+    AVLayerVideoGravity gravity = AVLayerVideoGravityResizeAspect;
+
+    switch (aspectRatioMode()) {
+    case Qt::IgnoreAspectRatio:
+        gravity = AVLayerVideoGravityResize;
+        break;
+    case Qt::KeepAspectRatio:
+        gravity = AVLayerVideoGravityResizeAspect;
+        break;
+    case Qt::KeepAspectRatioByExpanding:
+        gravity = AVLayerVideoGravityResizeAspectFill;
+        break;
+    default:
+        break;
+    }
+    auto *layer = static_cast<AVCaptureVideoPreviewLayer *>(m_layer);
+    layer.videoGravity = gravity;
+}
+
 #include "moc_avfcamerarenderercontrol_p.cpp"
diff --git a/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol_p.h b/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol_p.h
index 9dab72014..7c94b198f 100644
--- a/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol_p.h
+++ b/src/multimedia/platform/darwin/camera/avfcamerarenderercontrol_p.h
@@ -54,6 +54,7 @@
 #include <QtCore/qobject.h>
 #include <QtMultimedia/qvideoframe.h>
 #include <QtCore/qmutex.h>
+#include <private/avfvideosink_p.h>
 
 #include <dispatch/dispatch.h>
 
@@ -68,14 +69,15 @@ class AVFCameraRendererControl;
 class QAbstractVideoSurface;
 class AVFVideoSink;
 
-class AVFCameraRendererControl : public QObject
+class AVFCameraRendererControl : public QObject, public AVFVideoSinkInterface
 {
 Q_OBJECT
 public:
     AVFCameraRendererControl(QObject *parent = nullptr);
     ~AVFCameraRendererControl();
 
-    void setVideoSink(AVFVideoSink *sink);
+    void reconfigure() override;
+    void updateAspectRatio() override;
 
     void configureAVCaptureSession(AVFCameraSession *cameraSession);
     void syncHandleViewfinderFrame(const QVideoFrame &frame);
@@ -95,7 +97,6 @@ private Q_SLOTS:
     void updateCaptureConnection();
 
 private:
-    AVFVideoSink *m_sink = nullptr;
     AVFCaptureFramesDelegate *m_viewfinderFramesDelegate = nullptr;
     AVFCameraSession *m_cameraSession = nullptr;
     AVCaptureVideoDataOutput *m_videoDataOutput = nullptr;
@@ -109,6 +110,7 @@ private:
 
     QVideoFrame m_lastViewfinderFrame;
     QMutex m_vfMutex;
+    bool m_rendersToWindow = false;
     dispatch_queue_t m_delegateQueue;
 
     friend class CVImageVideoBuffer;
diff --git a/src/multimedia/platform/darwin/camera/avfcamerasession.mm b/src/multimedia/platform/darwin/camera/avfcamerasession.mm
index 100f0145c..b91e8a41a 100644
--- a/src/multimedia/platform/darwin/camera/avfcamerasession.mm
+++ b/src/multimedia/platform/darwin/camera/avfcamerasession.mm
@@ -352,10 +352,7 @@ void AVFCameraSession::setVideoSink(QVideoSink *sink)
     if (m_videoSink) {
         m_videoOutput->setVideoSink(videoSink);
         AVCaptureVideoPreviewLayer *previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:m_captureSession];
-        m_videoSink->setLayer(previewLayer);
-//        if (auto *camera = m_service->cameraControl()) {
-//            m_videoSink->setNativeSize(camera->viewfinderSettings().resolution());
-//        }
+        m_videoOutput->setLayer(previewLayer);
     }
 }
 
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
index 6d89de338..d0fcfe00a 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
@@ -130,17 +130,7 @@ void AVFVideoRendererControl::reconfigure()
     }
 
     updateAspectRatio();
-    //updateLayerBounds();
-    updateNativeSize();
-}
-
-void AVFVideoRendererControl::setLayer(CALayer *layer)
-{
-    if (m_layer == layer)
-        return;
-    AVFVideoSinkInterface::setLayer(layer);
-
-    reconfigure();
+    nativeSizeChanged();
 }
 
 void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
@@ -161,7 +151,7 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
     auto *layer = playerLayer();
     if (!layer.readyForDisplay || !m_frameRenderer)
         return;
-    updateNativeSize();
+    nativeSizeChanged();
 
     QVideoFrame frame;
     if (type == QVideoSink::Metal || type == QVideoSink::NativeTexture) {
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
index fc918ba2a..b7ed02f51 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
@@ -77,7 +77,6 @@ public:
     // AVFVideoSinkInterface
     void reconfigure() override;
     void updateAspectRatio() override;
-    void setLayer(CALayer *layer) override;
 
 private Q_SLOTS:
     void updateVideoFrame(const CVTimeStamp &ts);
diff --git a/src/multimediawidgets/qvideowidget.cpp b/src/multimediawidgets/qvideowidget.cpp
index 3ca846899..0cf449faf 100644
--- a/src/multimediawidgets/qvideowidget.cpp
+++ b/src/multimediawidgets/qvideowidget.cpp
@@ -108,7 +108,7 @@ QVideoWidget::QVideoWidget(QWidget *parent)
 {
     d_ptr->q_ptr = this;
     d_ptr->videoSink = new QVideoSink(this);
-//    d_ptr->videoSink->setGraphicsType(QVideoSink::NativeWindow);
+    d_ptr->videoSink->setGraphicsType(QVideoSink::NativeWindow);
     d_ptr->videoSink->setTargetRect(rect());
 
     d_ptr->videoSink->setNativeWindowId(winId());