AVFVideoRenderer: Properly update video output when changing media

When changing the video source of the AVFMediaPlayer, the video frame
was not updated and displayed the last frame of the previous video.
To fix, before setting a new CALayer to the renderer, make sure to
remove the video output and subtitle output from the previous
AVPlayerItem and add them to the updated one.
Also, clear the last video frame from the video output by setting a
null QVideoFrame() to the sink whenever the mediaplayer stops or
reaches end of stream.

Fixes: QTBUG-96368
Pick-to: 6.2.0 6.2
Change-Id: I74ed056c1ece0db319e857b89eaf5e5168a972a1
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

4 files changed, 38 insertions, 6 deletions

diff --git a/src/multimedia/platform/darwin/avfvideosink_p.h b/src/multimedia/platform/darwin/avfvideosink_p.h
index 4227bcb53..d25efdfc7 100644
--- a/src/multimedia/platform/darwin/avfvideosink_p.h
+++ b/src/multimedia/platform/darwin/avfvideosink_p.h
@@ -101,7 +101,7 @@ public:
     virtual void setRhi(QRhi *);
     QRhi *rhi() const { return m_rhi; }
 
-    void setLayer(CALayer *layer);
+    virtual void setLayer(CALayer *layer);
 
     void updateLayerBounds();
     void nativeSizeChanged() { updateLayerBounds(); }
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm b/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm
index c0904fddd..2cd61a42e 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfmediaplayer.mm
@@ -183,6 +183,8 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM
         [m_player release];
         m_player = 0;
     }
+    if (m_playerLayer)
+        m_playerLayer.player = nil;
 #if defined(Q_OS_IOS)
     [[AVAudioSession sharedInstance] setActive:NO error:nil];
 #endif
@@ -874,6 +876,9 @@ void AVFMediaPlayer::stop()
     [[static_cast<AVFMediaPlayerObserver*>(m_observer) player] pause];
     setPosition(0);
 
+    if (m_videoOutput)
+        m_videoOutput->setLayer(nullptr);
+
     if (m_mediaStatus == QMediaPlayer::BufferedMedia)
         Q_EMIT mediaStatusChanged((m_mediaStatus = QMediaPlayer::LoadedMedia));
 
@@ -928,6 +933,9 @@ void AVFMediaPlayer::processEOS()
     m_mediaStatus = QMediaPlayer::EndOfMedia;
     m_state = QMediaPlayer::StoppedState;
 
+    if (m_videoOutput)
+        m_videoOutput->setLayer(nullptr);
+
     Q_EMIT mediaStatusChanged(m_mediaStatus);
     Q_EMIT stateChanged(m_state);
 }
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
index a3d438e38..2eda39936 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
@@ -134,6 +134,26 @@ void AVFVideoRendererControl::reconfigure()
     nativeSizeChanged();
 }
 
+void AVFVideoRendererControl::setLayer(CALayer *layer)
+{
+    if (m_layer == layer)
+        return;
+
+    AVPlayerLayer *plLayer = playerLayer();
+    if (plLayer) {
+        if (m_videoOutput)
+            [[[plLayer player] currentItem] removeOutput:m_videoOutput];
+
+        if (m_subtitleOutput)
+            [[[plLayer player] currentItem] removeOutput:m_subtitleOutput];
+    }
+
+    if (!layer && m_sink)
+        m_sink->setVideoFrame(QVideoFrame());
+
+    AVFVideoSinkInterface::setLayer(layer);
+}
+
 void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
 {
     Q_UNUSED(ts);
@@ -141,10 +161,8 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
     if (!m_sink)
         return;
 
-    if (!m_layer) {
-        qWarning("updateVideoFrame called without AVPlayerLayer (which shouldn't happen");
+    if (!m_layer)
         return;
-    }
 
     auto *layer = playerLayer();
     if (!layer.readyForDisplay)
@@ -205,17 +223,22 @@ CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width
         return nullptr;
     }
 
+    AVPlayerItem * item = [[layer player] currentItem];
+
     if (!m_videoOutput) {
         auto *settings = (m_rhi && m_rhi->backend() == QRhi::OpenGLES2) ? AVF_OUTPUT_SETTINGS_OPENGL : AVF_OUTPUT_SETTINGS;
         m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:settings];
         [m_videoOutput setDelegate:nil queue:nil];
+    }
+    if (!m_subtitleOutput) {
         m_subtitleOutput = [[AVPlayerItemLegibleOutput alloc] init];
         m_subtitleDelegate = [[SubtitleDelegate alloc] initWithRenderer:this];
         [m_subtitleOutput setDelegate:m_subtitleDelegate queue:dispatch_get_main_queue()];
-        AVPlayerItem * item = [[layer player] currentItem];
+    }
+    if (![item.outputs containsObject:m_videoOutput])
         [item addOutput:m_videoOutput];
+    if (![item.outputs containsObject:m_subtitleOutput])
         [item addOutput:m_subtitleOutput];
-    }
 
     CFTimeInterval currentCAFrameTime =  CACurrentMediaTime();
     CMTime currentCMFrameTime =  [m_videoOutput itemTimeForHostTime:currentCAFrameTime];
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
index ff4846962..ce0ec0738 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h
@@ -78,6 +78,7 @@ public:
 
     // AVFVideoSinkInterface
     void reconfigure() override;
+    void setLayer(CALayer *layer) override;
 
     void setSubtitleText(const QString &subtitle)
     {