Add AVFCameraWindowControl as QVideoWindowControl_iid implementation

AVFCameraWindowControl is a renderer backend,
when QCamera viewfinder output set to QVideoWidget on macOS or iOS.
As it uses native API and lets the OS handle rendering pipeline,
it is an efficient alternative to default renderer backend.

Task-number: QTBUG-68779
Change-Id: I4d27022655a7d9f4ad7cf99c84b570a0c48cf10a
Reviewed-by: VaL Doroshchuk <valentyn.doroshchuk@qt.io>

7 files changed, 455 insertions, 3 deletions

diff --git a/src/plugins/avfoundation/camera/avfcameraservice.h b/src/plugins/avfoundation/camera/avfcameraservice.h
index 2969882b0..ec2884217 100644
--- a/src/plugins/avfoundation/camera/avfcameraservice.h
+++ b/src/plugins/avfoundation/camera/avfcameraservice.h
@@ -70,6 +70,7 @@ class AVFMediaRecorderControlIOS;
 class AVFAudioEncoderSettingsControl;
 class AVFVideoEncoderSettingsControl;
 class AVFMediaContainerControl;
+class AVFCameraWindowControl;
 
 class AVFCameraService : public QMediaService
 {
@@ -120,6 +121,7 @@ private:
     AVFAudioEncoderSettingsControl *m_audioEncoderSettingsControl;
     AVFVideoEncoderSettingsControl *m_videoEncoderSettingsControl;
     AVFMediaContainerControl *m_mediaContainerControl;
+    AVFCameraWindowControl *m_captureWindowControl;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/avfoundation/camera/avfcameraservice.mm b/src/plugins/avfoundation/camera/avfcameraservice.mm
index 33b4b72aa..627ecf67c 100644
--- a/src/plugins/avfoundation/camera/avfcameraservice.mm
+++ b/src/plugins/avfoundation/camera/avfcameraservice.mm
@@ -61,6 +61,7 @@
 #include "avfaudioencodersettingscontrol.h"
 #include "avfvideoencodersettingscontrol.h"
 #include "avfmediacontainercontrol.h"
+#include "avfcamerawindowcontrol.h"
 
 #ifdef Q_OS_IOS
 #include "avfcamerazoomcontrol.h"
@@ -74,7 +75,8 @@ QT_USE_NAMESPACE
 
 AVFCameraService::AVFCameraService(QObject *parent):
     QMediaService(parent),
-    m_videoOutput(nullptr)
+    m_videoOutput(nullptr),
+    m_captureWindowControl(nullptr)
 {
     m_session = new AVFCameraSession(this);
     m_cameraControl = new AVFCameraControl(this);
@@ -119,6 +121,12 @@ AVFCameraService::~AVFCameraService()
     delete m_recorderControl;
 #endif
 
+    if (m_captureWindowControl) {
+        m_session->setCapturePreviewOutput(nullptr);
+        delete m_captureWindowControl;
+        m_captureWindowControl = nullptr;
+    }
+
     if (m_videoOutput) {
         m_session->setVideoOutput(nullptr);
         delete m_videoOutput;
@@ -210,6 +218,14 @@ QMediaControl *AVFCameraService::requestControl(const char *name)
         return m_cameraZoomControl;
 #endif
 
+    if (!m_captureWindowControl) {
+        if (qstrcmp(name, QVideoWindowControl_iid) == 0) {
+            m_captureWindowControl = new AVFCameraWindowControl(this);
+            m_session->setCapturePreviewOutput(m_captureWindowControl);
+            return m_captureWindowControl;
+        }
+    }
+
     if (!m_videoOutput) {
         if (qstrcmp(name, QVideoRendererControl_iid) == 0)
             m_videoOutput = new AVFCameraRendererControl(this);
@@ -234,6 +250,11 @@ void AVFCameraService::releaseControl(QMediaControl *control)
         delete m_videoOutput;
         m_videoOutput = nullptr;
     }
+    else if (m_captureWindowControl == control) {
+        m_session->setCapturePreviewOutput(nullptr);
+        delete m_captureWindowControl;
+        m_captureWindowControl = nullptr;
+    }
 }
 
 
diff --git a/src/plugins/avfoundation/camera/avfcamerasession.h b/src/plugins/avfoundation/camera/avfcamerasession.h
index 103ec0e17..a449bb806 100644
--- a/src/plugins/avfoundation/camera/avfcamerasession.h
+++ b/src/plugins/avfoundation/camera/avfcamerasession.h
@@ -54,6 +54,7 @@ class AVFCameraControl;
 class AVFCameraService;
 class AVFCameraRendererControl;
 class AVFMediaVideoProbeControl;
+class AVFCameraWindowControl;
 
 struct AVFCameraInfo
 {
@@ -79,6 +80,7 @@ public:
     AVFCameraInfo activeCameraInfo() const { return m_activeCameraInfo; }
 
     void setVideoOutput(AVFCameraRendererControl *output);
+    void setCapturePreviewOutput(AVFCameraWindowControl *output);
     AVCaptureSession *captureSession() const { return m_captureSession; }
     AVCaptureDevice *videoCaptureDevice() const;
 
@@ -122,6 +124,7 @@ private:
 
     AVFCameraService *m_service;
     AVFCameraRendererControl *m_videoOutput;
+    AVFCameraWindowControl *m_capturePreviewWindowOutput;
 
     QCamera::State m_state;
     bool m_active;
diff --git a/src/plugins/avfoundation/camera/avfcamerasession.mm b/src/plugins/avfoundation/camera/avfcamerasession.mm
index 3c5f8f09a..6ee9c2636 100644
--- a/src/plugins/avfoundation/camera/avfcamerasession.mm
+++ b/src/plugins/avfoundation/camera/avfcamerasession.mm
@@ -48,6 +48,7 @@
 #include "avfcameraviewfindersettingscontrol.h"
 #include "avfimageencodercontrol.h"
 #include "avfcamerautility.h"
+#include "avfcamerawindowcontrol.h"
 
 #include <CoreFoundation/CoreFoundation.h>
 #include <Foundation/Foundation.h>
@@ -146,6 +147,7 @@ QList<AVFCameraInfo> AVFCameraSession::m_cameraDevices;
 AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent)
    : QObject(parent)
    , m_service(service)
+   , m_capturePreviewWindowOutput(nullptr)
    , m_state(QCamera::UnloadedState)
    , m_active(false)
    , m_videoInput(nil)
@@ -160,6 +162,10 @@ AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent)
 
 AVFCameraSession::~AVFCameraSession()
 {
+    if (m_capturePreviewWindowOutput) {
+        m_capturePreviewWindowOutput->setLayer(nil);
+    }
+
     if (m_videoInput) {
         [m_captureSession removeInput:m_videoInput];
         [m_videoInput release];
@@ -257,6 +263,29 @@ void AVFCameraSession::setVideoOutput(AVFCameraRendererControl *output)
         output->configureAVCaptureSession(this);
 }
 
+void AVFCameraSession::setCapturePreviewOutput(AVFCameraWindowControl *output)
+{
+#ifdef QT_DEBUG_AVF
+    qDebug() << Q_FUNC_INFO << output;
+#endif
+
+    if (m_capturePreviewWindowOutput == output)
+        return;
+
+    if (m_capturePreviewWindowOutput)
+        m_capturePreviewWindowOutput->setLayer(nil);
+
+    m_capturePreviewWindowOutput = output;
+
+    if (m_capturePreviewWindowOutput) {
+        AVCaptureVideoPreviewLayer *previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:m_captureSession];
+        m_capturePreviewWindowOutput->setLayer(previewLayer);
+        if (AVFCameraViewfinderSettingsControl2 *vfControl = m_service->viewfinderSettingsControl2()) {
+            m_capturePreviewWindowOutput->setNativeSize(vfControl->viewfinderSettings().resolution());
+        }
+    }
+}
+
 AVCaptureDevice *AVFCameraSession::videoCaptureDevice() const
 {
     if (m_videoInput)
@@ -409,6 +438,10 @@ bool AVFCameraSession::applyViewfinderSettings()
         }
 
         vfControl->applySettings(vfSettings);
+
+        if (m_capturePreviewWindowOutput)
+            m_capturePreviewWindowOutput->setNativeSize(vfControl->viewfinderSettings().resolution());
+
         return !vfSettings.isNull();
     }
 
diff --git a/src/plugins/avfoundation/camera/avfcamerawindowcontrol.h b/src/plugins/avfoundation/camera/avfcamerawindowcontrol.h
new file mode 100644
index 000000000..d1a950e38
--- /dev/null
+++ b/src/plugins/avfoundation/camera/avfcamerawindowcontrol.h
@@ -0,0 +1,129 @@
+/****************************************************************************
+**
+** Copyright (C) 2020 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef AVFCAMERAWINDOWCONTROL_H
+#define AVFCAMERAWINDOWCONTROL_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QVideoWindowControl>
+
+@class AVCaptureVideoPreviewLayer;
+#if defined(Q_OS_MACOS)
+@class NSView;
+typedef NSView NativeView;
+#else
+@class UIView;
+typedef UIView NativeView;
+#endif
+
+QT_BEGIN_NAMESPACE
+
+class AVFCameraWindowControl : public QVideoWindowControl
+{
+    Q_OBJECT
+public:
+    AVFCameraWindowControl(QObject *parent = nullptr);
+    virtual ~AVFCameraWindowControl() override;
+
+    // QVideoWindowControl interface
+public:
+    WId winId() const override;
+    void setWinId(WId id) override;
+
+    QRect displayRect() const override;
+    void setDisplayRect(const QRect &rect) override;
+
+    bool isFullScreen() const override;
+    void setFullScreen(bool fullScreen) override;
+
+    void repaint() override;
+
+    QSize nativeSize() const override;
+
+    Qt::AspectRatioMode aspectRatioMode() const override;
+    void setAspectRatioMode(Qt::AspectRatioMode mode) override;
+
+    int brightness() const override;
+    void setBrightness(int brightness) override;
+
+    int contrast() const override;
+    void setContrast(int contrast) override;
+
+    int hue() const override;
+    void setHue(int hue) override;
+
+    int saturation() const override;
+    void setSaturation(int saturation) override;
+
+    // AVF Camera implementation details
+    void setNativeSize(QSize size);
+    void setLayer(AVCaptureVideoPreviewLayer *capturePreviewLayer);
+
+private:
+    void updateAspectRatio();
+    void updateCaptureLayerBounds();
+
+    void retainNativeLayer();
+    void releaseNativeLayer();
+
+    void attachNativeLayer();
+    void detachNativeLayer();
+
+    WId m_winId{0};
+    QRect m_displayRect;
+    bool m_fullscreen{false};
+    Qt::AspectRatioMode m_aspectRatioMode{Qt::IgnoreAspectRatio};
+    QSize m_nativeSize;
+    AVCaptureVideoPreviewLayer *m_captureLayer{nullptr};
+    NativeView *m_nativeView{nullptr};
+};
+
+QT_END_NAMESPACE
+
+#endif // AVFCAMERAWINDOWCONTROL_H
diff --git a/src/plugins/avfoundation/camera/avfcamerawindowcontrol.mm b/src/plugins/avfoundation/camera/avfcamerawindowcontrol.mm
new file mode 100644
index 000000000..5154d0646
--- /dev/null
+++ b/src/plugins/avfoundation/camera/avfcamerawindowcontrol.mm
@@ -0,0 +1,262 @@
+/****************************************************************************
+**
+** Copyright (C) 2020 The Qt Company Ltd and/or its subsidiary(-ies).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "avfcamerawindowcontrol.h"
+
+#import <AVFoundation/AVFoundation.h>
+#import <QuartzCore/CATransaction.h>
+
+#if QT_HAS_INCLUDE(<AppKit/AppKit.h>)
+#import <AppKit/AppKit.h>
+#endif
+
+#if QT_HAS_INCLUDE(<UIKit/UIKit.h>)
+#import <UIKit/UIKit.h>
+#endif
+
+QT_USE_NAMESPACE
+
+AVFCameraWindowControl::AVFCameraWindowControl(QObject *parent)
+    : QVideoWindowControl(parent)
+{
+    setObjectName(QStringLiteral("AVFCameraWindowControl"));
+}
+
+AVFCameraWindowControl::~AVFCameraWindowControl()
+{
+    releaseNativeLayer();
+}
+
+WId AVFCameraWindowControl::winId() const
+{
+    return m_winId;
+}
+
+void AVFCameraWindowControl::setWinId(WId id)
+{
+    if (m_winId == id)
+        return;
+
+    m_winId = id;
+
+    detachNativeLayer();
+    m_nativeView = (NativeView*)m_winId;
+    attachNativeLayer();
+}
+
+QRect AVFCameraWindowControl::displayRect() const
+{
+    return m_displayRect;
+}
+
+void AVFCameraWindowControl::setDisplayRect(const QRect &rect)
+{
+    if (m_displayRect != rect) {
+        m_displayRect = rect;
+        updateCaptureLayerBounds();
+    }
+}
+
+bool AVFCameraWindowControl::isFullScreen() const
+{
+    return m_fullscreen;
+}
+
+void AVFCameraWindowControl::setFullScreen(bool fullscreen)
+{
+    if (m_fullscreen != fullscreen) {
+        m_fullscreen = fullscreen;
+        Q_EMIT fullScreenChanged(fullscreen);
+    }
+}
+
+void AVFCameraWindowControl::repaint()
+{
+    if (m_captureLayer)
+        [m_captureLayer setNeedsDisplay];
+}
+
+QSize AVFCameraWindowControl::nativeSize() const
+{
+    return m_nativeSize;
+}
+
+void AVFCameraWindowControl::setNativeSize(QSize size)
+{
+    if (m_nativeSize != size) {
+        m_nativeSize = size;
+        Q_EMIT nativeSizeChanged();
+    }
+}
+
+Qt::AspectRatioMode AVFCameraWindowControl::aspectRatioMode() const
+{
+    return m_aspectRatioMode;
+}
+
+void AVFCameraWindowControl::setAspectRatioMode(Qt::AspectRatioMode mode)
+{
+    if (m_aspectRatioMode != mode) {
+        m_aspectRatioMode = mode;
+        updateAspectRatio();
+    }
+}
+
+int AVFCameraWindowControl::brightness() const
+{
+    return 0;
+}
+
+void AVFCameraWindowControl::setBrightness(int brightness)
+{
+    if (0 != brightness)
+        qWarning("AVFCameraWindowControl doesn't support changing Brightness");
+}
+
+int AVFCameraWindowControl::contrast() const
+{
+    return 0;
+}
+
+void AVFCameraWindowControl::setContrast(int contrast)
+{
+    if (0 != contrast)
+        qWarning("AVFCameraWindowControl doesn't support changing Contrast");
+}
+
+int AVFCameraWindowControl::hue() const
+{
+    return 0;
+}
+
+void AVFCameraWindowControl::setHue(int hue)
+{
+    if (0 != hue)
+        qWarning("AVFCameraWindowControl doesn't support changing Hue");
+}
+
+int AVFCameraWindowControl::saturation() const
+{
+    return 0;
+}
+
+void AVFCameraWindowControl::setSaturation(int saturation)
+{
+    if (0 != saturation)
+        qWarning("AVFCameraWindowControl doesn't support changing Saturation");
+}
+
+void AVFCameraWindowControl::setLayer(AVCaptureVideoPreviewLayer *capturePreviewLayer)
+{
+    if (m_captureLayer == capturePreviewLayer)
+        return;
+
+    releaseNativeLayer();
+
+    m_captureLayer = capturePreviewLayer;
+
+    if (m_captureLayer)
+        retainNativeLayer();
+}
+
+void AVFCameraWindowControl::updateAspectRatio()
+{
+    if (m_captureLayer) {
+        switch (m_aspectRatioMode) {
+        case Qt::IgnoreAspectRatio:
+            [m_captureLayer setVideoGravity:AVLayerVideoGravityResize];
+            break;
+        case Qt::KeepAspectRatio:
+            [m_captureLayer setVideoGravity:AVLayerVideoGravityResizeAspect];
+            break;
+        case Qt::KeepAspectRatioByExpanding:
+            [m_captureLayer setVideoGravity:AVLayerVideoGravityResizeAspectFill];
+            break;
+        default:
+            break;
+        }
+    }
+}
+
+void AVFCameraWindowControl::updateCaptureLayerBounds()
+{
+    if (m_captureLayer && m_nativeView) {
+        [CATransaction begin];
+        [CATransaction setDisableActions: YES]; // disable animation/flicks
+        m_captureLayer.frame = m_displayRect.toCGRect();
+        [CATransaction commit];
+    }
+}
+
+void AVFCameraWindowControl::retainNativeLayer()
+{
+    [m_captureLayer retain];
+
+    updateAspectRatio();
+    attachNativeLayer();
+}
+
+void AVFCameraWindowControl::releaseNativeLayer()
+{
+    if (m_captureLayer) {
+        detachNativeLayer();
+        [m_captureLayer release];
+        m_captureLayer = nullptr;
+    }
+}
+
+void AVFCameraWindowControl::attachNativeLayer()
+{
+    if (m_captureLayer && m_nativeView) {
+#if defined(Q_OS_MACOS)
+        m_nativeView.wantsLayer = YES;
+#endif
+        CALayer *nativeLayer = m_nativeView.layer;
+        [nativeLayer addSublayer:m_captureLayer];
+        updateCaptureLayerBounds();
+    }
+}
+
+void AVFCameraWindowControl::detachNativeLayer()
+{
+    if (m_captureLayer && m_nativeView)
+        [m_captureLayer removeFromSuperlayer];
+}
+
+#include "moc_avfcamerawindowcontrol.cpp"
diff --git a/src/plugins/avfoundation/camera/camera.pro b/src/plugins/avfoundation/camera/camera.pro
index 4b960ed5a..70caf7536 100644
--- a/src/plugins/avfoundation/camera/camera.pro
+++ b/src/plugins/avfoundation/camera/camera.pro
@@ -42,7 +42,8 @@ HEADERS += \
     avfcameraflashcontrol.h \
     avfvideoencodersettingscontrol.h \
     avfmediacontainercontrol.h \
-    avfaudioencodersettingscontrol.h
+    avfaudioencodersettingscontrol.h \
+    avfcamerawindowcontrol.h
 
 OBJECTIVE_SOURCES += \
     avfcameraserviceplugin.mm \
@@ -65,7 +66,8 @@ OBJECTIVE_SOURCES += \
     avfcameraflashcontrol.mm \
     avfvideoencodersettingscontrol.mm \
     avfmediacontainercontrol.mm \
-    avfaudioencodersettingscontrol.mm
+    avfaudioencodersettingscontrol.mm \
+    avfcamerawindowcontrol.mm
 
 osx {