AVF: Introduce adoption of AVAssetResourceLoaderDelegate protocol

AVAssetResourceLoaderDelegate allows to load custom resources.

Implemented resourceLoader:shouldWaitForLoadingOfRequestedResource to read data
from QIODevice.

The device should be seekable, and already should have all data available.
Since there is a need to know total size of the stream.

So the media player will wait for QIODevice::readyRead before loading the resource.

Also it requires to have url together with the stream:
QMediaPlayer->setMedia(QUrl("does_not_matter.mp3"), buffer);
Since the backend uses extension to determine type of the stream.

Fixes: QTBUG-69101
Change-Id: I8ab0b69f668ccd67c42a8e5d5c1ad518d3306cce
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

4 files changed, 128 insertions, 13 deletions

diff --git a/src/multimedia/playback/qmediaplayer.cpp b/src/multimedia/playback/qmediaplayer.cpp
index 2f1aa45c3..9d26e8d38 100644
--- a/src/multimedia/playback/qmediaplayer.cpp
+++ b/src/multimedia/playback/qmediaplayer.cpp
@@ -1005,8 +1005,9 @@ void QMediaPlayer::setPlaybackRate(qreal rate)
     Sets the current \a media source.
 
     If a \a stream is supplied; media data will be read from it instead of resolving the media
-    source. In this case the media source may still be used to resolve additional information
+    source. In this case the url should be provided to resolve additional information
     about the media such as mime type. The \a stream must be open and readable.
+    For macOS the \a stream should be also seekable.
 
     Setting the media to a null QMediaContent will cause the player to discard all
     information relating to the current media source and to cease all I/O operations related
diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h
index 7a268a3d9..db29e88aa 100644
--- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h
+++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h
@@ -67,7 +67,7 @@ public:
     QMediaPlayer::MediaStatus mediaStatus() const;
 
     QMediaContent media() const;
-    const QIODevice *mediaStream() const;
+    QIODevice *mediaStream() const;
     void setMedia(const QMediaContent &content, QIODevice *stream);
 
     qint64 position() const;
@@ -110,6 +110,9 @@ public Q_SLOTS:
 
     void processDurationChange(qint64 duration);
 
+    void streamReady();
+    void streamDestroyed();
+
 Q_SIGNALS:
     void positionChanged(qint64 position);
     void durationChanged(qint64 duration);
@@ -128,6 +131,7 @@ private:
     void setAudioAvailable(bool available);
     void setVideoAvailable(bool available);
     void setSeekable(bool seekable);
+    void resetStream(QIODevice *stream = nullptr);
 
     AVFMediaPlayerService *m_service;
     AVFVideoOutput *m_videoOutput;
diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm
index 1f13fc9bb..6a2dba6a0 100644
--- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm
+++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm
@@ -42,6 +42,7 @@
 #include "avfvideooutput.h"
 
 #include <qpointer.h>
+#include <QFileInfo>
 
 #import <AVFoundation/AVFoundation.h>
 
@@ -66,7 +67,7 @@ static void *AVFMediaPlayerSessionObserverBufferLikelyToKeepUpContext = &AVFMedi
 static void *AVFMediaPlayerSessionObserverCurrentItemObservationContext = &AVFMediaPlayerSessionObserverCurrentItemObservationContext;
 static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext = &AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext;
 
-@interface AVFMediaPlayerSessionObserver : NSObject
+@interface AVFMediaPlayerSessionObserver : NSObject<AVAssetResourceLoaderDelegate>
 
 @property (readonly, getter=player) AVPlayer* m_player;
 @property (readonly, getter=playerItem) AVPlayerItem* m_playerItem;
@@ -74,7 +75,7 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext
 @property (readonly, getter=session) AVFMediaPlayerSession* m_session;
 
 - (AVFMediaPlayerSessionObserver *) initWithMediaPlayerSession:(AVFMediaPlayerSession *)session;
-- (void) setURL:(NSURL *)url;
+- (void) setURL:(NSURL *)url mimeType:(NSString *)mimeType;
 - (void) unloadMedia;
 - (void) prepareToPlayAsset:(AVURLAsset *)asset withKeys:(NSArray *)requestedKeys;
 - (void) assetFailedToPrepareForPlayback:(NSError *)error;
@@ -84,6 +85,7 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext
                          change:(NSDictionary *)change context:(void *)context;
 - (void) detatchSession;
 - (void) dealloc;
+- (BOOL) resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest;
 @end
 
 @implementation AVFMediaPlayerSessionObserver
@@ -95,6 +97,8 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext
     AVPlayerLayer *m_playerLayer;
     NSURL *m_URL;
     BOOL m_bufferIsLikelyToKeepUp;
+    NSData *m_data;
+    NSString *m_mimeType;
 }
 
 @synthesize m_player, m_playerItem, m_playerLayer, m_session;
@@ -109,8 +113,11 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext
     return self;
 }
 
-- (void) setURL:(NSURL *)url
+- (void) setURL:(NSURL *)url mimeType:(NSString *)mimeType
 {
+    [m_mimeType release];
+    m_mimeType = [mimeType retain];
+
     if (m_URL != url)
     {
         [m_URL release];
@@ -122,6 +129,8 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext
         // use __block to avoid maintaining strong references on variables captured by the
         // following block callback
         __block AVURLAsset *asset = [[AVURLAsset URLAssetWithURL:m_URL options:nil] retain];
+        [asset.resourceLoader setDelegate:self queue:dispatch_get_main_queue()];
+
         __block NSArray *requestedKeys = [[NSArray arrayWithObjects:AVF_TRACKS_KEY, AVF_PLAYABLE_KEY, nil] retain];
 
         __block AVFMediaPlayerSessionObserver *blockSelf = self;
@@ -403,9 +412,48 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext
         [m_URL release];
     }
 
+    [m_mimeType release];
     [super dealloc];
 }
 
+- (BOOL) resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest
+{
+    Q_UNUSED(resourceLoader);
+
+    if (![loadingRequest.request.URL.scheme isEqualToString:@"iodevice"])
+        return NO;
+
+    QIODevice *device = m_session->mediaStream();
+    if (!device)
+        return NO;
+
+    device->seek(loadingRequest.dataRequest.requestedOffset);
+    if (loadingRequest.contentInformationRequest) {
+        loadingRequest.contentInformationRequest.contentType = m_mimeType;
+        loadingRequest.contentInformationRequest.contentLength = device->size();
+        loadingRequest.contentInformationRequest.byteRangeAccessSupported = YES;
+    }
+
+    if (loadingRequest.dataRequest) {
+        NSInteger requestedLength = loadingRequest.dataRequest.requestedLength;
+        int maxBytes = qMin(32 * 1064, int(requestedLength));
+        char buffer[maxBytes];
+        NSInteger submitted = 0;
+        while (submitted < requestedLength) {
+            qint64 len = device->read(buffer, maxBytes);
+            if (len < 1)
+                break;
+
+            [loadingRequest.dataRequest respondWithData:[NSData dataWithBytes:buffer length:len]];
+            submitted += len;
+        }
+
+        // Finish loading even if not all bytes submitted.
+        [loadingRequest finishLoading];
+    }
+
+    return YES;
+}
 @end
 
 AVFMediaPlayerSession::AVFMediaPlayerSession(AVFMediaPlayerService *service, QObject *parent)
@@ -483,11 +531,23 @@ QMediaContent AVFMediaPlayerSession::media() const
     return m_resources;
 }
 
-const QIODevice *AVFMediaPlayerSession::mediaStream() const
+QIODevice *AVFMediaPlayerSession::mediaStream() const
 {
     return m_mediaStream;
 }
 
+static void setURL(void *observer, const QString &url, const QString &mimeType = QString())
+{
+    NSString *urlString = [NSString stringWithUTF8String:url.toUtf8().constData()];
+    NSURL *nsurl = [NSURL URLWithString:urlString];
+    [static_cast<AVFMediaPlayerSessionObserver*>(observer) setURL:nsurl mimeType:[NSString stringWithUTF8String:mimeType.toLatin1().constData()]];
+}
+
+static void setStreamURL(void *observer, const QString &url)
+{
+    setURL(observer, QLatin1String("iodevice://") + url, QFileInfo(url).suffix());
+}
+
 void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *stream)
 {
 #ifdef QT_DEBUG_AVF
@@ -497,7 +557,7 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st
     [static_cast<AVFMediaPlayerSessionObserver*>(m_observer) unloadMedia];
 
     m_resources = content;
-    m_mediaStream = stream;
+    resetStream(stream);
 
     setAudioAvailable(false);
     setVideoAvailable(false);
@@ -508,7 +568,7 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st
     const QMediaPlayer::MediaStatus oldMediaStatus = m_mediaStatus;
     const QMediaPlayer::State oldState = m_state;
 
-    if (content.isNull() || content.request().url().isEmpty()) {
+    if (!m_mediaStream && (content.isNull() || content.request().url().isEmpty())) {
         m_mediaStatus = QMediaPlayer::NoMedia;
         if (m_mediaStatus != oldMediaStatus)
             Q_EMIT mediaStatusChanged(m_mediaStatus);
@@ -524,11 +584,16 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st
     if (m_mediaStatus != oldMediaStatus)
         Q_EMIT mediaStatusChanged(m_mediaStatus);
 
-    //Load AVURLAsset
-    //initialize asset using content's URL
-    NSString *urlString = [NSString stringWithUTF8String:content.request().url().toEncoded().constData()];
-    NSURL *url = [NSURL URLWithString:urlString];
-    [static_cast<AVFMediaPlayerSessionObserver*>(m_observer) setURL:url];
+    if (m_mediaStream) {
+        // If there is a data, try to load it,
+        // otherwise wait for readyRead.
+        if (m_mediaStream->size())
+            setStreamURL(m_observer, m_resources.request().url().toString());
+    } else {
+        //Load AVURLAsset
+        //initialize asset using content's URL
+        setURL(m_observer, m_resources.request().url().toString());
+    }
 
     m_state = QMediaPlayer::StoppedState;
     if (m_state != oldState)
@@ -967,3 +1032,28 @@ void AVFMediaPlayerSession::processMediaLoadError()
 
     Q_EMIT error(QMediaPlayer::FormatError, tr("Failed to load media"));
 }
+
+void AVFMediaPlayerSession::streamReady()
+{
+    setStreamURL(m_observer, m_resources.request().url().toString());
+}
+
+void AVFMediaPlayerSession::streamDestroyed()
+{
+    resetStream(nullptr);
+}
+
+void AVFMediaPlayerSession::resetStream(QIODevice *stream)
+{
+    if (m_mediaStream) {
+        disconnect(m_mediaStream, &QIODevice::readyRead, this, &AVFMediaPlayerSession::streamReady);
+        disconnect(m_mediaStream, &QIODevice::destroyed, this, &AVFMediaPlayerSession::streamDestroyed);
+    }
+
+    m_mediaStream = stream;
+
+    if (m_mediaStream) {
+        connect(m_mediaStream, &QIODevice::readyRead, this, &AVFMediaPlayerSession::streamReady);
+        connect(m_mediaStream, &QIODevice::destroyed, this, &AVFMediaPlayerSession::streamDestroyed);
+    }
+}
diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
index f1be070e8..a2217afc8 100644
--- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
+++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
@@ -81,6 +81,7 @@ private slots:
     void multipleSurfaces();
     void metadata();
     void playerStateAtEOS();
+    void playFromBuffer();
 
 private:
     QMediaContent selectVideoFile(const QStringList& mediaCandidates);
@@ -1476,6 +1477,25 @@ void tst_QMediaPlayerBackend::playerStateAtEOS()
     QVERIFY(endOfMediaReceived);
 }
 
+void tst_QMediaPlayerBackend::playFromBuffer()
+{
+    if (localVideoFile.isNull())
+        QSKIP("No supported video file");
+
+    TestVideoSurface surface(false);
+    QMediaPlayer player;
+    player.setVideoOutput(&surface);
+    QFile file(localVideoFile.request().url().toLocalFile());
+    if (!file.open(QIODevice::ReadOnly))
+        QSKIP("Could not open file");
+    player.setMedia(localVideoFile, &file);
+    player.play();
+    QTRY_VERIFY(player.position() >= 1000);
+    if (surface.error() == QAbstractVideoSurface::UnsupportedFormatError)
+        QSKIP("None of the pixel formats is supported by the backend");
+    QVERIFY2(surface.m_totalFrames >= 25, qPrintable(QString("Expected >= 25, got %1").arg(surface.m_totalFrames)));
+}
+
 TestVideoSurface::TestVideoSurface(bool storeFrames):
     m_totalFrames(0),
     m_storeFrames(storeFrames)