Further refactor the video sink implementation on macOS/iOS.

Make sure we can deliver both SW buffers, textures and render onto
native surfaces.

As we need different implementations for the player and capture
pipelines, AVFVideoSink is now acting as a proxy to the class
doing the actual rendering.

Fold the windowed mode handling into the AVFVideoRenderer class and use
it when rendering windowed or fullscreen.

This also required some changes in AVFMediaPlayer to ensure that we always
have a AVPlayerLayer available because loading content happends asynchronously,
but we need the layer directly.

With this playback works both onto QVideoWidget and the QML VideoItem, although
there are some R<->B issues when rendering using software.

Change-Id: Ie6fe901f35989c5076eedaea4edf3b8c4d3f35f4
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

9 files changed, 264 insertions, 275 deletions

diff --git a/src/multimedia/platform/darwin/avfvideosink.mm b/src/multimedia/platform/darwin/avfvideosink.mm
index 4b048d953..c7473c70a 100644
--- a/src/multimedia/platform/darwin/avfvideosink.mm
+++ b/src/multimedia/platform/darwin/avfvideosink.mm
@@ -59,10 +59,18 @@ AVFVideoSink::AVFVideoSink(QVideoSink *parent)
 
 AVFVideoSink::~AVFVideoSink()
 {
-    if (m_layer) {
-        [m_layer removeFromSuperlayer];
-        [m_layer release];
-    }
+}
+
+bool AVFVideoSink::setGraphicsType(QVideoSink::GraphicsType type)
+{
+    if (type == m_graphicsType)
+        return true;
+    if (type == QVideoSink::Direct3D11 || type == QVideoSink::Vulkan)
+        return false;
+    m_graphicsType = type;
+    if (m_interface)
+        m_interface->reconfigure();
+    return true;
 }
 
 WId AVFVideoSink::winId() const
@@ -72,8 +80,12 @@ WId AVFVideoSink::winId() const
 
 void AVFVideoSink::setWinId(WId id)
 {
+    if (id == m_winId)
+        return;
     m_winId = id;
     m_nativeView = (NativeView*)m_winId;
+    if (m_interface)
+        m_interface->reconfigure();
 }
 
 QRect AVFVideoSink::displayRect() const
@@ -83,10 +95,11 @@ QRect AVFVideoSink::displayRect() const
 
 void AVFVideoSink::setDisplayRect(const QRect &rect)
 {
-    if (m_displayRect != rect) {
-        m_displayRect = rect;
-        updatePlayerLayerBounds();
-    }
+    if (m_displayRect == rect)
+        return;
+    m_displayRect = rect;
+    if (m_interface)
+        m_interface->updateLayerBounds();
 }
 
 bool AVFVideoSink::isFullScreen() const
@@ -96,13 +109,15 @@ bool AVFVideoSink::isFullScreen() const
 
 void AVFVideoSink::setFullScreen(bool fullScreen)
 {
+    if (fullScreen == m_fullscreen)
+        return;
     m_fullscreen = fullScreen;
+    if (m_interface)
+        m_interface->reconfigure();
 }
 
 void AVFVideoSink::repaint()
 {
-    if (m_layer)
-        [m_layer setNeedsDisplay];
 }
 
 QSize AVFVideoSink::nativeSize() const
@@ -110,6 +125,15 @@ QSize AVFVideoSink::nativeSize() const
     return m_nativeSize;
 }
 
+void AVFVideoSink::setNativeSize(QSize size)
+{
+    if (size == m_nativeSize)
+        return;
+    m_nativeSize = size;
+    if (m_interface)
+        m_interface->updateNativeSize();
+}
+
 Qt::AspectRatioMode AVFVideoSink::aspectRatioMode() const
 {
     return m_aspectRatioMode;
@@ -117,10 +141,11 @@ Qt::AspectRatioMode AVFVideoSink::aspectRatioMode() const
 
 void AVFVideoSink::setAspectRatioMode(Qt::AspectRatioMode mode)
 {
-    if (m_aspectRatioMode != mode) {
-        m_aspectRatioMode = mode;
-        updateAspectRatio();
-    }
+    if (m_aspectRatioMode == mode)
+        return;
+    m_aspectRatioMode = mode;
+    if (m_interface)
+        m_interface->updateAspectRatio();
 }
 
 int AVFVideoSink::brightness() const
@@ -163,82 +188,72 @@ void AVFVideoSink::setSaturation(int saturation)
     m_saturation = saturation;
 }
 
-template<typename T> inline T* objc_cast(id from) {
-    if ([from isKindOfClass:[T class]]) {
-        return static_cast<T*>(from);
-    }
-    return nil;
+void AVFVideoSink::setLayer(CALayer *)
+{
 }
 
-void AVFVideoSink::setLayer(CALayer *layer)
+void AVFVideoSink::setVideoSinkInterface(AVFVideoSinkInterface *interface)
 {
-    m_playerLayer = objc_cast<AVPlayerLayer>(layer);
-    m_previewLayer = objc_cast<AVCaptureVideoPreviewLayer>(layer);
-    if (m_layer == layer)
-        return;
+    m_interface = interface;
+}
 
-    if (!m_winId) {
-        qDebug("AVFVideoSink: No video window");
+AVFVideoSinkInterface::~AVFVideoSinkInterface()
+{
+    if (m_layer)
+        [m_layer release];
+}
+
+void AVFVideoSinkInterface::setVideoSink(AVFVideoSink *sink)
+{
+    if (sink == m_sink)
         return;
-    }
 
-#if defined(Q_OS_OSX)
-    [m_nativeView setWantsLayer:YES];
-#endif
+    m_sink = sink;
+    if (m_sink)
+        m_sink->setVideoSinkInterface(this);
+    reconfigure();
+}
+#include <qdebug.h>
 
+void AVFVideoSinkInterface::setLayer(CALayer *layer)
+{
     if (m_layer) {
-        [m_layer removeFromSuperlayer];
+        renderToNativeView(false);
         [m_layer release];
     }
-
     m_layer = layer;
-
-    CALayer *nativeLayer = [m_nativeView layer];
-
-    if (layer) {
-        [layer retain];
-
-        m_nativeSize = QSize(m_layer.bounds.size.width,
-                             m_layer.bounds.size.height);
-
-        updateAspectRatio();
-        [nativeLayer addSublayer:m_layer];
-        updatePlayerLayerBounds();
-    }
+    [m_layer retain];
 }
 
-void AVFVideoSink::updateAspectRatio()
+void AVFVideoSinkInterface::renderToNativeView(bool enable)
 {
-    AVLayerVideoGravity gravity = AVLayerVideoGravityResizeAspect;
-
-    switch (m_aspectRatioMode) {
-    case Qt::IgnoreAspectRatio:
-        gravity = AVLayerVideoGravityResize;
-        break;
-    case Qt::KeepAspectRatio:
-        gravity = AVLayerVideoGravityResizeAspect;
-        break;
-    case Qt::KeepAspectRatioByExpanding:
-        gravity = AVLayerVideoGravityResizeAspectFill;
-        break;
-    default:
-        break;
+    auto *view = nativeView();
+    if (enable && view && m_layer) {
+        CALayer *nativeLayer = [view layer];
+        [nativeLayer addSublayer:m_layer];
+        m_rendersToWindow = true;
+    } else {
+        if (m_layer)
+            [m_layer removeFromSuperlayer];
+        m_rendersToWindow = false;
     }
-    if (m_playerLayer)
-        m_playerLayer.videoGravity = gravity;
-    else if (m_previewLayer)
-        m_previewLayer.videoGravity = gravity;
+    updateLayerBounds();
 }
-#include <qdebug.h>
 
-void AVFVideoSink::updatePlayerLayerBounds()
+void AVFVideoSinkInterface::updateLayerBounds()
 {
-    if (m_layer) {
-        [CATransaction begin];
-        [CATransaction setDisableActions: YES]; // disable animation/flicks
-        m_layer.frame = m_displayRect.toCGRect();
-        [CATransaction commit];
+    if (!m_layer)
+        return;
+    [CATransaction begin];
+    [CATransaction setDisableActions: YES]; // disable animation/flicks
+    if (m_rendersToWindow) {
+        m_layer.frame = displayRect().toCGRect();
+    } else {
+        m_layer.frame = QRectF(0, 0, nativeSize().width(), nativeSize().height()).toCGRect();
+        m_layer.bounds = m_layer.frame;
     }
+    [CATransaction commit];
 }
 
+
 #include "moc_avfvideosink_p.cpp"
diff --git a/src/multimedia/platform/darwin/avfvideosink_p.h b/src/multimedia/platform/darwin/avfvideosink_p.h
index 159c519d7..c5b953aca 100644
--- a/src/multimedia/platform/darwin/avfvideosink_p.h
+++ b/src/multimedia/platform/darwin/avfvideosink_p.h
@@ -66,6 +66,8 @@ typedef UIView NativeView;
 
 QT_BEGIN_NAMESPACE
 
+class AVFVideoSinkInterface;
+
 class AVFVideoSink : public QPlatformVideoSink
 {
     Q_OBJECT
@@ -76,6 +78,9 @@ public:
 
     // QPlatformVideoSink interface
 public:
+    QVideoSink::GraphicsType graphicsType() const override { return m_graphicsType; }
+    bool setGraphicsType(QVideoSink::GraphicsType type) override;
+
     WId winId() const override;
     void setWinId(WId id) override;
 
@@ -87,6 +92,7 @@ public:
 
     void repaint() override;
     QSize nativeSize() const override;
+    void setNativeSize(QSize size);
 
     Qt::AspectRatioMode aspectRatioMode() const override;
     void setAspectRatioMode(Qt::AspectRatioMode mode) override;
@@ -105,11 +111,16 @@ public:
 
     void setLayer(CALayer *playerLayer);
 
-private:
-    void updateAspectRatio();
-    void updatePlayerLayerBounds();
+    void setVideoSinkInterface(AVFVideoSinkInterface *interface);
+    NativeView *nativeView() const { return m_nativeView; }
 
+private:
+    AVFVideoSinkInterface *m_interface = nullptr;
+    QVideoSink::GraphicsType m_graphicsType = QVideoSink::Memory;
     WId m_winId = 0;
+    NativeView *m_nativeView = nullptr;
+
+    QSize m_nativeSize;
     QRect m_displayRect;
     bool m_fullscreen = false;
     int m_brightness = 0;
@@ -117,13 +128,46 @@ private:
     int m_hue = 0;
     int m_saturation = 0;
     Qt::AspectRatioMode m_aspectRatioMode = Qt::KeepAspectRatio;
-    QSize m_nativeSize;
-    AVCaptureVideoPreviewLayer *m_previewLayer = nullptr;
-    AVPlayerLayer *m_playerLayer = nullptr;
+};
+
+class AVFVideoSinkInterface
+{
+public:
+    ~AVFVideoSinkInterface();
+
+    void setVideoSink(AVFVideoSink *sink);
+
+
+    virtual void reconfigure() = 0;
+    virtual void updateAspectRatio() = 0;
+    virtual void setLayer(CALayer *layer);
+
+    void renderToNativeView(bool enable);
+
+    bool shouldRenderToWindow()
+    {
+        return m_layer && nativeView() && (graphicsType() == QVideoSink::NativeWindow || isFullScreen());
+    }
+    bool rendersToWindow() const { return m_rendersToWindow; }
+
+    void updateLayerBounds();
+    void updateNativeSize() { updateLayerBounds(); }
+
+protected:
+    NativeView *nativeView() const { return m_sink->nativeView(); }
+    QRect displayRect() { return m_sink->displayRect(); }
+    Qt::AspectRatioMode aspectRatioMode() const { return m_sink->aspectRatioMode(); }
+    QVideoSink::GraphicsType graphicsType() const { return m_sink->graphicsType(); }
+    bool isFullScreen() const { return m_sink->isFullScreen(); }
+    QSize nativeSize() const { return m_sink->nativeSize(); }
+
+    AVFVideoSink *m_sink = nullptr;
     CALayer *m_layer = nullptr;
-    NativeView *m_nativeView = nullptr;
+    QSize m_nativeSize;
+    bool m_rendersToWindow = false;
 };
 
+
 QT_END_NAMESPACE
 
 #endif // AVFVIDEOWINDOWCONTROL_H
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm b/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm
index b00a970f0..f283291ae 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm
@@ -65,6 +65,7 @@ static NSString* const AVF_CURRENT_ITEM_DURATION_KEY    = @"currentItem.duration
 
 static void *AVFMediaPlayerObserverRateObservationContext = &AVFMediaPlayerObserverRateObservationContext;
 static void *AVFMediaPlayerObserverStatusObservationContext = &AVFMediaPlayerObserverStatusObservationContext;
+static void *AVFMediaPlayerObserverPresentationSizeContext = &AVFMediaPlayerObserverPresentationSizeContext;
 static void *AVFMediaPlayerObserverBufferLikelyToKeepUpContext = &AVFMediaPlayerObserverBufferLikelyToKeepUpContext;
 static void *AVFMediaPlayerObserverTracksContext = &AVFMediaPlayerObserverTracksContext;
 static void *AVFMediaPlayerObserverCurrentItemObservationContext = &AVFMediaPlayerObserverCurrentItemObservationContext;
@@ -112,8 +113,13 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
     if (!(self = [super init]))
         return nil;
 
-    self->m_session = session;
-    self->m_bufferIsLikelyToKeepUp = FALSE;
+    m_session = session;
+    m_bufferIsLikelyToKeepUp = FALSE;
+
+    m_playerLayer = [AVPlayerLayer playerLayerWithPlayer:nil];
+    [m_playerLayer retain];
+    m_playerLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
+    m_playerLayer.anchorPoint = CGPointMake(0.0f, 0.0f);
     return self;
 }
 
@@ -157,6 +163,7 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
 - (void) unloadMedia
 {
     if (m_playerItem) {
+        [m_playerItem removeObserver:self forKeyPath:@"presentationSize"];
         [m_playerItem removeObserver:self forKeyPath:AVF_STATUS_KEY];
         [m_playerItem removeObserver:self forKeyPath:AVF_BUFFER_LIKELY_KEEP_UP_KEY];
         [m_playerItem removeObserver:self forKeyPath:AVF_TRACKS_KEY];
@@ -177,10 +184,6 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
         [m_player release];
         m_player = 0;
     }
-    if (m_playerLayer) {
-        [m_playerLayer release];
-        m_playerLayer = 0;
-    }
 #if defined(Q_OS_IOS)
     [[AVAudioSession sharedInstance] setActive:NO error:nil];
 #endif
@@ -242,6 +245,11 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
                       context:AVFMediaPlayerObserverStatusObservationContext];
 
     [m_playerItem addObserver:self
+                   forKeyPath:@"presentationSize"
+                      options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
+                      context:AVFMediaPlayerObserverPresentationSizeContext];
+
+    [m_playerItem addObserver:self
                    forKeyPath:AVF_BUFFER_LIKELY_KEEP_UP_KEY
                       options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                       context:AVFMediaPlayerObserverBufferLikelyToKeepUpContext];
@@ -273,14 +281,8 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
         [m_player setMuted:m_session->isMuted()];
     }
 
-    //Create a new player layer if we don't have one already
-    if (!m_playerLayer)
-    {
-        m_playerLayer = [AVPlayerLayer playerLayerWithPlayer:m_player];
-        [m_playerLayer retain];
-        m_playerLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
-        m_playerLayer.anchorPoint = CGPointMake(0.0f, 0.0f);
-    }
+    //Assign the output layer to the new player
+    m_playerLayer.player = m_player;
 
     //Observe the AVPlayer "currentItem" property to find out when any
     //AVPlayer replaceCurrentItemWithPlayerItem: replacement will/did
@@ -372,8 +374,10 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
             }
             break;
         }
-    }
-    else if (context == AVFMediaPlayerObserverBufferLikelyToKeepUpContext)
+    } else if (context == AVFMediaPlayerObserverPresentationSizeContext) {
+        QSize size(m_playerItem.presentationSize.width, m_playerItem.presentationSize.height);
+        QMetaObject::invokeMethod(m_session, "nativeSizeChanged", Qt::AutoConnection, Q_ARG(QSize, size));
+    } else if (context == AVFMediaPlayerObserverBufferLikelyToKeepUpContext)
     {
         const bool isPlaybackLikelyToKeepUp = [m_playerItem isPlaybackLikelyToKeepUp];
         if (isPlaybackLikelyToKeepUp != m_bufferIsLikelyToKeepUp) {
@@ -433,6 +437,7 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
     }
 
     [m_mimeType release];
+    [m_playerLayer release];
     [super dealloc];
 }
 
@@ -507,15 +512,14 @@ AVFMediaPlayer::~AVFMediaPlayer()
     [static_cast<AVFMediaPlayerObserver*>(m_observer) release];
 }
 
-void AVFMediaPlayer::setVideoSurface(QAbstractVideoSurface *surface)
+void AVFMediaPlayer::setVideoSurface(QAbstractVideoSurface *)
 {
-    m_videoOutput->setSurface(surface);
 }
 
 void AVFMediaPlayer::setVideoSink(QVideoSink *sink)
 {
-    setVideoOutput(nullptr);
     m_videoSink = static_cast<AVFVideoSink *>(sink->platformVideoSink());
+    m_videoOutput->setVideoSink(m_videoSink);
 }
 
 void AVFMediaPlayer::setVideoOutput(AVFVideoRendererControl *output)
@@ -827,11 +831,8 @@ void AVFMediaPlayer::play()
     if (m_state == QMediaPlayer::PlayingState)
         return;
 
-    if (m_videoSink) {
-        m_videoSink->setLayer([static_cast<AVFMediaPlayerObserver*>(m_observer) playerLayer]);
-    } else if (m_videoOutput) {
+    if (m_videoOutput)
         m_videoOutput->setLayer([static_cast<AVFMediaPlayerObserver*>(m_observer) playerLayer]);
-    }
 
     // Reset media status if the current status is EndOfMedia
     if (m_mediaStatus == QMediaPlayer::EndOfMedia)
@@ -862,11 +863,8 @@ void AVFMediaPlayer::pause()
 
     m_state = QMediaPlayer::PausedState;
 
-    if (m_videoSink) {
-        m_videoSink->setLayer([static_cast<AVFMediaPlayerObserver*>(m_observer) playerLayer]);
-    } else if (m_videoOutput) {
+    if (m_videoOutput)
         m_videoOutput->setLayer([static_cast<AVFMediaPlayerObserver*>(m_observer) playerLayer]);
-    }
 
     [[static_cast<AVFMediaPlayerObserver*>(m_observer) player] pause];
 
@@ -891,11 +889,8 @@ void AVFMediaPlayer::stop()
     [[static_cast<AVFMediaPlayerObserver*>(m_observer) player] pause];
     setPosition(0);
 
-    if (m_videoSink) {
+    if (m_videoSink)
         m_videoSink->setLayer(nullptr);
-    } else if (m_videoOutput) {
-        m_videoOutput->setLayer(nullptr);
-    }
 
     if (m_mediaStatus == QMediaPlayer::BufferedMedia)
         Q_EMIT mediaStatusChanged((m_mediaStatus = QMediaPlayer::LoadedMedia));
@@ -952,11 +947,8 @@ void AVFMediaPlayer::processEOS()
 
     // At this point, frames should not be rendered anymore.
     // Clear the output layer to make sure of that.
-    if (m_videoSink) {
+    if (m_videoSink)
         m_videoSink->setLayer(nullptr);
-    } else if (m_videoOutput) {
-        m_videoOutput->setLayer(nullptr);
-    }
 
     Q_EMIT mediaStatusChanged(m_mediaStatus);
     Q_EMIT stateChanged(m_state);
@@ -997,11 +989,8 @@ void AVFMediaPlayer::processLoadStateChange(QMediaPlayer::State newState)
                 }
 
                 if (newState != QMediaPlayer::StoppedState) {
-                    if (m_videoSink) {
+                    if (m_videoSink)
                         m_videoSink->setLayer(playerLayer);
-                    } else if (m_videoOutput) {
-                        m_videoOutput->setLayer(playerLayer);
-                    }
                 }
             }
 
@@ -1182,3 +1171,9 @@ void AVFMediaPlayer::resetStream(QIODevice *stream)
         connect(m_mediaStream, &QIODevice::destroyed, this, &AVFMediaPlayer::streamDestroyed);
     }
 }
+
+void AVFMediaPlayer::nativeSizeChanged(QSize size)
+{
+    qDebug() << "presentation size" << size;
+    m_videoSink->setNativeSize(size);
+}
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer_p.h b/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer_p.h
index c2cc928a3..6788b6f10 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer_p.h
+++ b/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer_p.h
@@ -111,6 +111,7 @@ public:
 
 public Q_SLOTS:
     void setPlaybackRate(qreal rate) override;
+    void nativeSizeChanged(QSize size);
 
     void setPosition(qint64 pos) override;
 
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm
index 88c83f1a8..aca969713 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm
@@ -53,10 +53,10 @@
 
 QT_USE_NAMESPACE
 
-AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent)
+AVFVideoFrameRenderer::AVFVideoFrameRenderer(QObject *parent)
     : QObject(parent)
     , m_videoLayerRenderer(nullptr)
-    , m_surface(surface)
+    , m_surface(nullptr)
     , m_offscreenSurface(nullptr)
     , m_glContext(nullptr)
     , m_currentBuffer(1)
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h
index 465c1e563..8c8815d62 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h
@@ -73,7 +73,7 @@ class QAbstractVideoSurface;
 class AVFVideoFrameRenderer : public QObject
 {
 public:
-    AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent = nullptr);
+    AVFVideoFrameRenderer(QObject *parent = nullptr);
 
     virtual ~AVFVideoFrameRenderer();
 
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
index 0c2ce11e2..6d89de338 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
@@ -51,6 +51,7 @@
 #include <QtMultimedia/qvideosurfaceformat.h>
 
 #include <private/qimagevideobuffer_p.h>
+#include <private/avfvideosink_p.h>
 
 #include <QtCore/qdebug.h>
 
@@ -85,12 +86,6 @@ private:
 
 AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent)
     : QObject(parent)
-    , m_surface(nullptr)
-    , m_playerLayer(nullptr)
-    , m_frameRenderer(nullptr)
-    , m_enableOpenGL(false)
-    , m_enableMetal(false)
-
 {
     m_displayLink = new AVFDisplayLink(this);
     connect(m_displayLink, SIGNAL(tick(CVTimeStamp)), SLOT(updateVideoFrame(CVTimeStamp)));
@@ -102,201 +97,144 @@ AVFVideoRendererControl::~AVFVideoRendererControl()
     qDebug() << Q_FUNC_INFO;
 #endif
     m_displayLink->stop();
-    [static_cast<AVPlayerLayer*>(m_playerLayer) release];
-}
-
-QAbstractVideoSurface *AVFVideoRendererControl::surface() const
-{
-    return m_surface;
 }
 
-void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface)
+void AVFVideoRendererControl::reconfigure()
 {
 #ifdef QT_DEBUG_AVF
-    qDebug() << "Set video surface" << surface;
+    qDebug() << "reconfigure";
 #endif
-
-    //When we have a valid surface, we can setup a frame renderer
-    //and schedule surface updates with the display link.
-    if (surface == m_surface)
+    if (!m_layer) {
+        m_displayLink->stop();
         return;
+    }
 
     QMutexLocker locker(&m_mutex);
 
-    if (m_surface && m_surface->isActive())
-        m_surface->stop();
+    renderToNativeView(shouldRenderToWindow());
 
-    m_surface = surface;
-
-    //If the surface changed, then the current frame renderer is no longer valid
-    delete m_frameRenderer;
-    m_frameRenderer = nullptr;
-
-    //If there is now no surface to render too
-    if (m_surface == nullptr) {
+    if (rendersToWindow()) {
+        if (m_frameRenderer) {
+            delete m_frameRenderer;
+            m_frameRenderer = nullptr;
+        }
         m_displayLink->stop();
-        return;
-    }
-
-    //Surface changed, so we need a new frame renderer
-    m_frameRenderer = new AVFVideoFrameRenderer(m_surface, this);
+    } else {
+        if (!m_frameRenderer)
+            m_frameRenderer = new AVFVideoFrameRenderer(this);
 #if defined(Q_OS_IOS) || defined(Q_OS_TVOS)
-    if (m_playerLayer) {
-        m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(m_playerLayer));
-    }
+        if (m_layer)
+            m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(m_layer));
 #endif
-
-    auto checkHandleType = [this] {
-        m_enableOpenGL = m_surface->supportedPixelFormats(QVideoFrame::GLTextureHandle).contains(QVideoFrame::Format_BGR32);
-        m_enableMetal = m_surface->supportedPixelFormats(QVideoFrame::MTLTextureHandle).contains(QVideoFrame::Format_BGR32);
-    };
-    checkHandleType();
-    connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, checkHandleType);
-
-    //If we already have a layer, but changed surfaces start rendering again
-    if (m_playerLayer && !m_displayLink->isActive()) {
         m_displayLink->start();
     }
 
+    updateAspectRatio();
+    //updateLayerBounds();
+    updateNativeSize();
 }
 
-void AVFVideoRendererControl::setLayer(CALayer *playerLayer)
+void AVFVideoRendererControl::setLayer(CALayer *layer)
 {
-    if (m_playerLayer == playerLayer)
+    if (m_layer == layer)
         return;
+    AVFVideoSinkInterface::setLayer(layer);
 
-    m_playerLayer = [static_cast<AVPlayerLayer*>(playerLayer) retain];
-
-    //If there is an active surface, make sure it has been stopped so that
-    //we can update it's state with the new content.
-    if (m_surface && m_surface->isActive())
-        m_surface->stop();
-
-#if defined(Q_OS_IOS) || defined(Q_OS_TVOS)
-    if (m_frameRenderer) {
-        m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(playerLayer));
-    }
-#endif
-
-    //If there is no layer to render, stop scheduling updates
-    if (m_playerLayer == nullptr) {
-        m_displayLink->stop();
-        return;
-    }
-
-    setupVideoOutput();
-
-    //If we now have both a valid surface and layer, start scheduling updates
-    if (m_surface && !m_displayLink->isActive()) {
-        m_displayLink->start();
-    }
+    reconfigure();
 }
 
 void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
 {
     Q_UNUSED(ts);
 
-    AVPlayerLayer *playerLayer = static_cast<AVPlayerLayer*>(m_playerLayer);
+    auto type = graphicsType();
+    Q_ASSERT(type != QVideoSink::NativeWindow);
 
-    if (!playerLayer) {
+    if (!m_sink)
+        return;
+
+    if (!m_layer) {
         qWarning("updateVideoFrame called without AVPlayerLayer (which shouldn't happen");
         return;
     }
 
-    if (!playerLayer.readyForDisplay || !m_surface)
+    auto *layer = playerLayer();
+    if (!layer.readyForDisplay || !m_frameRenderer)
         return;
+    updateNativeSize();
 
-    if (m_enableMetal) {
-        quint64 tex = m_frameRenderer->renderLayerToMTLTexture(playerLayer);
+    QVideoFrame frame;
+    if (type == QVideoSink::Metal || type == QVideoSink::NativeTexture) {
+        quint64 tex = m_frameRenderer->renderLayerToMTLTexture(layer);
         if (tex == 0)
             return;
 
         auto buffer = new TextureVideoBuffer(QVideoFrame::MTLTextureHandle, tex);
-        QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32);
-        if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat())
-            m_surface->stop();
+        frame = QVideoFrame(buffer, nativeSize(), QVideoFrame::Format_BGR32);
+        if (!frame.isValid())
+            return;
 
-        if (!m_surface->isActive()) {
-            QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::MTLTextureHandle);
+        QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::MTLTextureHandle);
 #if defined(Q_OS_IOS) || defined(Q_OS_TVOS)
-            format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom);
+        format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom);
 #else
-            format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop);
+        format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop);
 #endif
-            if (!m_surface->start(format))
-                qWarning("Failed to activate video surface");
-        }
-
-        if (m_surface->isActive())
-            m_surface->present(frame);
-
-        return;
-    }
-
-    if (m_enableOpenGL) {
-        quint64 tex = m_frameRenderer->renderLayerToTexture(playerLayer);
+        // #### QVideoFrame needs the surfaceformat!
+    } else if (type == QVideoSink::OpenGL) {
+        quint64 tex = m_frameRenderer->renderLayerToTexture(layer);
         //Make sure we got a valid texture
         if (tex == 0)
             return;
 
         QAbstractVideoBuffer *buffer = new TextureVideoBuffer(QVideoFrame::GLTextureHandle, tex);
-        QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32);
-
-        if (m_surface && frame.isValid()) {
-            if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat())
-                m_surface->stop();
+        frame = QVideoFrame(buffer, nativeSize(), QVideoFrame::Format_BGR32);
+        if (!frame.isValid())
+            return;
 
-            if (!m_surface->isActive()) {
-                QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::GLTextureHandle);
+        QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::GLTextureHandle);
 #if defined(Q_OS_IOS) || defined(Q_OS_TVOS)
-                format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom);
+        format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom);
 #else
-                format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop);
+        format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop);
 #endif
-                if (!m_surface->start(format)) {
-                    //Surface doesn't support GLTextureHandle
-                    qWarning("Failed to activate video surface");
-                }
-            }
-
-            if (m_surface->isActive())
-                m_surface->present(frame);
-        }
     } else {
+        Q_ASSERT(type == QVideoSink::Memory);
         //fallback to rendering frames to QImages
-        QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer);
+        QImage frameData = m_frameRenderer->renderLayerToImage(layer);
 
-        if (frameData.isNull()) {
+        if (frameData.isNull())
             return;
-        }
 
         QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData);
-        QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32);
-        if (m_surface && frame.isValid()) {
-            if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat())
-                m_surface->stop();
-
-            if (!m_surface->isActive()) {
-                QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::NoHandle);
-
-                if (!m_surface->start(format)) {
-                    qWarning("Failed to activate video surface");
-                }
-            }
-
-            if (m_surface->isActive())
-                m_surface->present(frame);
-        }
-
+        frame = QVideoFrame(buffer, nativeSize(), QVideoFrame::Format_ARGB32_Premultiplied);
+        QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::NoHandle);
     }
+
+    m_sink->videoSink()->newVideoFrame(frame);
 }
 
-void AVFVideoRendererControl::setupVideoOutput()
+void AVFVideoRendererControl::updateAspectRatio()
 {
-    AVPlayerLayer *playerLayer = static_cast<AVPlayerLayer*>(m_playerLayer);
-    if (playerLayer)
-        m_nativeSize = QSize(playerLayer.bounds.size.width, playerLayer.bounds.size.height);
-}
+    if (!m_layer)
+        return;
 
+    AVLayerVideoGravity gravity = AVLayerVideoGravityResizeAspect;
+
+    switch (aspectRatioMode()) {
+    case Qt::IgnoreAspectRatio:
+        gravity = AVLayerVideoGravityResize;
+        break;
+    case Qt::KeepAspectRatio:
+        gravity = AVLayerVideoGravityResizeAspect;
+        break;
+    case Qt::KeepAspectRatioByExpanding:
+        gravity = AVLayerVideoGravityResizeAspectFill;
+        break;
+    default:
+        break;
+    }
+    playerLayer().videoGravity = gravity;
+}
 
 #include "moc_avfvideorenderercontrol_p.cpp"
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
index 3343fd8cd..fc918ba2a 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
@@ -55,6 +55,8 @@
 #include <QtCore/QMutex>
 #include <QtCore/QSize>
 
+#include <private/avfvideosink_p.h>
+
 #include <CoreVideo/CVBase.h>
 
 Q_FORWARD_DECLARE_OBJC_CLASS(CALayer);
@@ -65,17 +67,17 @@ class AVFDisplayLink;
 class AVFVideoFrameRenderer;
 class QAbstractVideoSurface;
 
-class AVFVideoRendererControl : public QObject
+class AVFVideoRendererControl : public QObject, public AVFVideoSinkInterface
 {
     Q_OBJECT
 public:
     explicit AVFVideoRendererControl(QObject *parent = nullptr);
     virtual ~AVFVideoRendererControl();
 
-    QAbstractVideoSurface *surface() const;
-    void setSurface(QAbstractVideoSurface *surface);
-
-    void setLayer(CALayer *playerLayer);
+    // AVFVideoSinkInterface
+    void reconfigure() override;
+    void updateAspectRatio() override;
+    void setLayer(CALayer *layer) override;
 
 private Q_SLOTS:
     void updateVideoFrame(const CVTimeStamp &ts);
@@ -84,18 +86,12 @@ Q_SIGNALS:
     void surfaceChanged(QAbstractVideoSurface *surface);
 
 private:
-    void setupVideoOutput();
+    AVPlayerLayer *playerLayer() const { return static_cast<AVPlayerLayer *>(m_layer); }
 
     QMutex m_mutex;
-    QAbstractVideoSurface *m_surface;
-
-    CALayer *m_playerLayer;
 
-    AVFVideoFrameRenderer *m_frameRenderer;
-    AVFDisplayLink *m_displayLink;
-    QSize m_nativeSize;
-    bool m_enableOpenGL;
-    bool m_enableMetal;
+    AVFVideoFrameRenderer *m_frameRenderer = nullptr;
+    AVFDisplayLink *m_displayLink = nullptr;
 };
 
 QT_END_NAMESPACE
diff --git a/src/multimediawidgets/qvideowidget.cpp b/src/multimediawidgets/qvideowidget.cpp
index 0cf449faf..3ca846899 100644
--- a/src/multimediawidgets/qvideowidget.cpp
+++ b/src/multimediawidgets/qvideowidget.cpp
@@ -108,7 +108,7 @@ QVideoWidget::QVideoWidget(QWidget *parent)
 {
     d_ptr->q_ptr = this;
     d_ptr->videoSink = new QVideoSink(this);
-    d_ptr->videoSink->setGraphicsType(QVideoSink::NativeWindow);
+//    d_ptr->videoSink->setGraphicsType(QVideoSink::NativeWindow);
     d_ptr->videoSink->setTargetRect(rect());
 
     d_ptr->videoSink->setNativeWindowId(winId());