Implement image capturing for ScreenCapture and WindowCapture

The idea is that frames observers, ImageCapture and MediaRecorder,
should communicate with QPlatformVideoSource, not with inheritors.

The approach has been already applied for CaptureSession
and MediaRecorder (in the previous patches).
The current patch applies the approach for ImageCapture with
additional cleanup, e.g. unnecessary stuff, like cameraActiveChange,
has been eliminated.

Change-Id: Ib354b2023cf6716bd1100419cadf888fc5489d97
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
(cherry picked from commit f0db56bde4875f4b4fb26f31f66441c3615e60cb)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

6 files changed, 88 insertions, 75 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp b/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp
index 44931fed9..9265b088d 100644
--- a/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp
+++ b/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp
@@ -19,7 +19,7 @@ int QAndroidImageCapture::doCapture(const QString &fileName)
 {
     auto ret = QFFmpegImageCapture::doCapture(fileName);
     if (ret >= 0) {
-        auto androidCamera = static_cast<QAndroidCamera *>(m_camera);
+        auto androidCamera = qobject_cast<QAndroidCamera *>(videoSource());
         if (androidCamera)
             androidCamera->capture();
     }
@@ -27,11 +27,11 @@ int QAndroidImageCapture::doCapture(const QString &fileName)
     return ret;
 }
 
-
-void QAndroidImageCapture::setupCameraConnections()
+void QAndroidImageCapture::setupVideoSourceConnections()
 {
-    connect(m_camera, &QPlatformCamera::activeChanged, this, &QFFmpegImageCapture::cameraActiveChanged);
-    auto androidCamera = static_cast<QAndroidCamera *>(m_camera);
+    auto androidCamera = qobject_cast<QAndroidCamera *>(videoSource());
     if (androidCamera)
         connect(androidCamera, &QAndroidCamera::onCaptured, this, &QAndroidImageCapture::newVideoFrame);
+    else
+        QFFmpegImageCapture::setupVideoSourceConnections();
 }
diff --git a/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h b/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h
index 686ae4201..ad9a9568b 100644
--- a/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h
+++ b/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h
@@ -27,7 +27,7 @@ public:
     ~QAndroidImageCapture() override;
 
 protected:
-    void setupCameraConnections() override;
+    void setupVideoSourceConnections() override;
     int doCapture(const QString &fileName) override;
 };
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp
index 45314edfe..2fb878784 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp
@@ -17,6 +17,9 @@
 
 QT_BEGIN_NAMESPACE
 
+// Probably, might be increased. To be investigated and tested on Android implementation
+static constexpr int MaxPendingImagesCount = 1;
+
 static Q_LOGGING_CATEGORY(qLcImageCapture, "qt.multimedia.imageCapture")
 
 QFFmpegImageCapture::QFFmpegImageCapture(QImageCapture *parent)
@@ -80,7 +83,7 @@ int QFFmpegImageCapture::doCapture(const QString &fileName)
         qCDebug(qLcImageCapture) << "error 1";
         return -1;
     }
-    if (!m_camera) {
+    if (!m_videoSource) {
         //emit error in the next event loop,
         //so application can associate it with returned request id.
         QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
@@ -91,7 +94,7 @@ int QFFmpegImageCapture::doCapture(const QString &fileName)
         qCDebug(qLcImageCapture) << "error 2";
         return -1;
     }
-    if (passImage) {
+    if (m_pendingImages.size() >= MaxPendingImagesCount) {
         //emit error in the next event loop,
         //so application can associate it with returned request id.
         QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
@@ -102,13 +105,12 @@ int QFFmpegImageCapture::doCapture(const QString &fileName)
         qCDebug(qLcImageCapture) << "error 3";
         return -1;
     }
-    m_lastId++;
 
-    pendingImages.enqueue({m_lastId, fileName, QMediaMetaData{}});
-    // let one image pass the pipeline
-    passImage = true;
+    m_lastId++;
 
+    m_pendingImages.enqueue({ m_lastId, fileName, QMediaMetaData{} });
     updateReadyForCapture();
+
     return m_lastId;
 }
 
@@ -119,48 +121,39 @@ void QFFmpegImageCapture::setCaptureSession(QPlatformMediaCaptureSession *sessio
         return;
 
     if (m_session) {
-        disconnect(m_session, nullptr, this, nullptr);
+        m_session->disconnect(this);
         m_lastId = 0;
-        pendingImages.clear();
-        passImage = false;
-        cameraActive = false;
+        m_pendingImages.clear();
     }
 
     m_session = captureSession;
+
     if (m_session)
-        connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, &QFFmpegImageCapture::onCameraChanged);
+        connect(m_session, &QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged, this,
+                &QFFmpegImageCapture::onVideoSourceChanged);
 
-    onCameraChanged();
-    updateReadyForCapture();
+    onVideoSourceChanged();
 }
 
 void QFFmpegImageCapture::updateReadyForCapture()
 {
-    bool ready = m_session && !passImage && cameraActive;
-    if (ready == m_isReadyForCapture)
-        return;
-    m_isReadyForCapture = ready;
-    emit readyForCaptureChanged(m_isReadyForCapture);
-}
+    const bool ready = m_session && m_pendingImages.size() < MaxPendingImagesCount && m_videoSource
+            && m_videoSource->isActive();
 
-void QFFmpegImageCapture::cameraActiveChanged(bool active)
-{
-    qCDebug(qLcImageCapture) << "cameraActiveChanged" << cameraActive << active;
-    if (cameraActive == active)
-        return;
-    cameraActive = active;
-    qCDebug(qLcImageCapture) << "isReady" << isReadyForCapture();
-    updateReadyForCapture();
+    qCDebug(qLcImageCapture) << "updateReadyForCapture" << ready;
+
+    if (std::exchange(m_isReadyForCapture, ready) != ready)
+        emit readyForCaptureChanged(ready);
 }
 
 void QFFmpegImageCapture::newVideoFrame(const QVideoFrame &frame)
 {
-    if (!passImage)
+    if (m_pendingImages.empty())
         return;
 
-    passImage = false;
-    Q_ASSERT(!pendingImages.isEmpty());
-    auto pending = pendingImages.dequeue();
+    auto pending = m_pendingImages.dequeue();
+
+    qCDebug(qLcImageCapture) << "Taking image" << pending.id;
 
     emit imageExposed(pending.id);
     // ### Add metadata from the AVFrame
@@ -218,32 +211,33 @@ void QFFmpegImageCapture::newVideoFrame(const QVideoFrame &frame)
             emit error(pending.id, err, writer.errorString());
         }
     }
+
     updateReadyForCapture();
 }
 
-void QFFmpegImageCapture::setupCameraConnections()
+void QFFmpegImageCapture::setupVideoSourceConnections()
 {
-    connect(m_camera, &QPlatformCamera::activeChanged, this, &QFFmpegImageCapture::cameraActiveChanged);
-    connect(m_camera, &QPlatformCamera::newVideoFrame, this, &QFFmpegImageCapture::newVideoFrame);
+    connect(m_videoSource, &QPlatformCamera::newVideoFrame, this,
+            &QFFmpegImageCapture::newVideoFrame);
 }
 
-void QFFmpegImageCapture::onCameraChanged()
+QPlatformVideoSource *QFFmpegImageCapture::videoSource() const
 {
-    auto *camera = m_session ? m_session->camera() : nullptr;
-    if (m_camera == camera)
-        return;
+    return m_videoSource;
+}
 
-    if (m_camera)
-        disconnect(m_camera);
+void QFFmpegImageCapture::onVideoSourceChanged()
+{
+    if (m_videoSource)
+        m_videoSource->disconnect(this);
 
-    m_camera = camera;
+    m_videoSource = m_session ? m_session->primaryActiveVideoSource() : nullptr;
 
-    if (m_camera) {
-        cameraActiveChanged(m_camera->isActive());
-        setupCameraConnections();
-    } else {
-        cameraActiveChanged(false);
-    }
+    // TODO: optimize, setup the connection only when the capture is ready
+    if (m_videoSource)
+        setupVideoSourceConnections();
+
+    updateReadyForCapture();
 }
 
 QImageEncoderSettings QFFmpegImageCapture::imageSettings() const
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h
index 8eef92cad..43faacf7d 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h
@@ -24,7 +24,6 @@
 QT_BEGIN_NAMESPACE
 
 class QFFmpegImageCapture : public QPlatformImageCapture
-
 {
     Q_OBJECT
 public:
@@ -40,20 +39,19 @@ public:
 
     void setCaptureSession(QPlatformMediaCaptureSession *session);
 
+protected:
+    virtual int doCapture(const QString &fileName);
+    virtual void setupVideoSourceConnections();
+    QPlatformVideoSource *videoSource() const;
     void updateReadyForCapture();
 
-public Q_SLOTS:
-    void cameraActiveChanged(bool active);
+protected Q_SLOTS:
     void newVideoFrame(const QVideoFrame &frame);
-    void onCameraChanged();
-
-protected:
-    virtual int doCapture(const QString &fileName);
-    virtual void setupCameraConnections();
-    QPlatformCamera *m_camera = nullptr;
+    void onVideoSourceChanged();
 
 private:
     QFFmpegMediaCaptureSession *m_session = nullptr;
+    QPointer<QPlatformVideoSource> m_videoSource;
     int m_lastId = 0;
     QImageEncoderSettings m_settings;
 
@@ -63,9 +61,7 @@ private:
         QMediaMetaData metaData;
     };
 
-    QQueue<PendingImage> pendingImages;
-    bool passImage = false;
-    bool cameraActive = false;
+    QQueue<PendingImage> m_pendingImages;
     bool m_isReadyForCapture = false;
 };
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp
index f62ecf4e7..5047f48c0 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp
@@ -31,12 +31,14 @@ static int preferredAudioSinkBufferSize(const QFFmpegAudioInput &input)
     return input.bufferSize() * BufferSizeFactor + BufferSizeExceeding;
 }
 
-QFFmpegMediaCaptureSession::QFFmpegMediaCaptureSession() = default;
-
-QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession()
+QFFmpegMediaCaptureSession::QFFmpegMediaCaptureSession()
 {
+    connect(this, &QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged, this,
+            &QFFmpegMediaCaptureSession::updateVideoFrameConnection);
 }
 
+QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession() = default;
+
 QPlatformCamera *QFFmpegMediaCaptureSession::camera()
 {
     return m_camera;
@@ -233,15 +235,24 @@ void QFFmpegMediaCaptureSession::updateVideoFrameConnection()
 {
     disconnect(m_videoFrameConnection);
 
-    if (auto sources = activeVideoSources(); !sources.empty() && m_videoSink) {
+    if (m_primaryActiveVideoSource && m_videoSink) {
         // deliver frames directly to video sink;
         // AutoConnection type might be a pessimization due to an extra queuing
         // TODO: investigate and integrate direct connection
-        m_videoFrameConnection = connect(sources.front(), &QPlatformVideoSource::newVideoFrame,
-                                         m_videoSink, &QVideoSink::setVideoFrame);
+        m_videoFrameConnection =
+                connect(m_primaryActiveVideoSource, &QPlatformVideoSource::newVideoFrame,
+                        m_videoSink, &QVideoSink::setVideoFrame);
     }
 }
 
+void QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource()
+{
+    auto sources = activeVideoSources();
+    auto source = sources.empty() ? nullptr : sources.front();
+    if (std::exchange(m_primaryActiveVideoSource, source) != source)
+        emit primaryActiveVideoSourceChanged();
+}
+
 template<typename VideoSource>
 bool QFFmpegMediaCaptureSession::setVideoSource(QPointer<VideoSource> &source,
                                                 VideoSource *newSource)
@@ -257,16 +268,21 @@ bool QFFmpegMediaCaptureSession::setVideoSource(QPointer<VideoSource> &source,
     if (source) {
         source->setCaptureSession(this);
         connect(source, &QPlatformVideoSource::activeChanged, this,
-                &QFFmpegMediaCaptureSession::updateVideoFrameConnection);
+                &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource);
         connect(source, &QObject::destroyed, this,
-                &QFFmpegMediaCaptureSession::updateVideoFrameConnection, Qt::QueuedConnection);
+                &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource, Qt::QueuedConnection);
     }
 
-    updateVideoFrameConnection();
+    updatePrimaryActiveVideoSource();
 
     return true;
 }
 
+QPlatformVideoSource *QFFmpegMediaCaptureSession::primaryActiveVideoSource()
+{
+    return m_primaryActiveVideoSource;
+}
+
 QT_END_NAMESPACE
 
 #include "moc_qffmpegmediacapturesession_p.cpp"
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h
index 044c5e9fc..6c80d0b09 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h
@@ -61,10 +61,16 @@ public:
     void setVideoPreview(QVideoSink *sink) override;
     void setAudioOutput(QPlatformAudioOutput *output) override;
 
-public Q_SLOTS:
+    QPlatformVideoSource *primaryActiveVideoSource();
+
+private Q_SLOTS:
     void updateAudioSink();
     void updateVolume();
     void updateVideoFrameConnection();
+    void updatePrimaryActiveVideoSource();
+
+Q_SIGNALS:
+    void primaryActiveVideoSourceChanged();
 
 private:
     template<typename VideoSource>
@@ -73,6 +79,7 @@ private:
     QPointer<QPlatformCamera> m_camera;
     QPointer<QPlatformSurfaceCapture> m_screenCapture;
     QPointer<QPlatformSurfaceCapture> m_windowCapture;
+    QPointer<QPlatformVideoSource> m_primaryActiveVideoSource;
 
     QFFmpegAudioInput *m_audioInput = nullptr;
     QFFmpegImageCapture *m_imageCapture = nullptr;