Add audio output selection to the capture session

Make it possible to select an audio output in the capture
session. This gives an audio monitor channel to monitor the
recording.

Currently only implemented on gstreamer.

Change-Id: I1da3d80c71253364c99ee49c770f8665527d01a0
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
Reviewed-by: André de la Rocha <andre.rocha@qt.io>

10 files changed, 112 insertions, 76 deletions

diff --git a/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput.cpp b/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput.cpp
index 4227a04d5..da0ffce72 100644
--- a/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput.cpp
+++ b/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput.cpp
@@ -40,6 +40,7 @@
 #include <private/qgstreameraudiooutput_p.h>
 #include <private/qgstreameraudiodevice_p.h>
 #include <qaudiodevice.h>
+#include <qaudiooutput.h>
 
 #include <QtCore/qloggingcategory.h>
 #include <QtNetwork/qnetworkaccessmanager.h>
@@ -53,9 +54,10 @@ Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput")
 
 QT_BEGIN_NAMESPACE
 
-QGstreamerAudioOutput::QGstreamerAudioOutput(QObject *parent)
-    : QObject(parent),
-      gstAudioOutput("audioOutput")
+QGstreamerAudioOutput::QGstreamerAudioOutput(QAudioOutput *parent)
+  : QObject(parent),
+    QPlatformAudioOutput(parent),
+    gstAudioOutput("audioOutput")
 {
     audioQueue = QGstElement("queue", "audioQueue");
     audioConvert = QGstElement("audioconvert", "audioConvert");
@@ -72,32 +74,14 @@ QGstreamerAudioOutput::~QGstreamerAudioOutput()
 {
 }
 
-int QGstreamerAudioOutput::volume() const
+void QGstreamerAudioOutput::setVolume(float vol)
 {
-    return m_volume;
-}
-
-bool QGstreamerAudioOutput::isMuted() const
-{
-    return m_muted;
-}
-
-void QGstreamerAudioOutput::setVolume(int vol)
-{
-    if (vol == m_volume)
-        return;
-    m_volume = vol;
-    audioVolume.set("volume", vol/100.);
-    emit volumeChanged(m_volume);
+    audioVolume.set("volume", vol);
 }
 
 void QGstreamerAudioOutput::setMuted(bool muted)
 {
-    if (muted == m_muted)
-        return;
-    m_muted = muted;
     audioVolume.set("mute", muted);
-    emit mutedChanged(muted);
 }
 
 void QGstreamerAudioOutput::setPipeline(const QGstPipeline &pipeline)
diff --git a/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput_p.h b/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput_p.h
index 940d82330..fc1a8b8db 100644
--- a/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput_p.h
+++ b/src/multimedia/platform/gstreamer/common/qgstreameraudiooutput_p.h
@@ -58,28 +58,28 @@
 
 #include <private/qgst_p.h>
 #include <private/qgstpipeline_p.h>
+#include <private/qplatformaudiooutput_p.h>
 
 QT_BEGIN_NAMESPACE
 
 class QGstreamerMessage;
 class QAudioDevice;
 
-class Q_MULTIMEDIA_EXPORT QGstreamerAudioOutput : public QObject
+class Q_MULTIMEDIA_EXPORT QGstreamerAudioOutput : public QObject, public QPlatformAudioOutput
 {
     Q_OBJECT
 
 public:
-    QGstreamerAudioOutput(QObject *parent = 0);
+    QGstreamerAudioOutput(QAudioOutput *parent);
     ~QGstreamerAudioOutput();
 
-    int volume() const;
-    bool isMuted() const;
-
     bool setAudioOutput(const QAudioDevice &);
     QAudioDevice audioOutput() const;
 
-    void setVolume(int volume);
-    void setMuted(bool muted);
+    void setAudioDevice(const QAudioDevice &) override
+    { setAudioOutput(device); }
+    void setVolume(float volume) override;
+    void setMuted(bool muted) override;
 
     void setPipeline(const QGstPipeline &pipeline);
 
@@ -93,9 +93,6 @@ private:
     void prepareAudioOutputChange(const QGstPad &pad);
     bool changeAudioOutput();
 
-    int m_volume = 100.;
-    bool m_muted = false;
-
     QAudioDevice m_audioOutput;
 
     // Gst elements
diff --git a/src/multimedia/platform/gstreamer/common/qgstreamermediaplayer.cpp b/src/multimedia/platform/gstreamer/common/qgstreamermediaplayer.cpp
index 63061961d..08ca8f23b 100644
--- a/src/multimedia/platform/gstreamer/common/qgstreamermediaplayer.cpp
+++ b/src/multimedia/platform/gstreamer/common/qgstreamermediaplayer.cpp
@@ -70,7 +70,7 @@ QGstreamerMediaPlayer::QGstreamerMediaPlayer(QMediaPlayer *parent)
       QPlatformMediaPlayer(parent),
       playerPipeline("playerPipeline")
 {
-    gstAudioOutput = new QGstreamerAudioOutput(this);
+    gstAudioOutput = new QGstreamerAudioOutput(nullptr);
     gstAudioOutput->setPipeline(playerPipeline);
     connect(gstAudioOutput, &QGstreamerAudioOutput::mutedChanged, this, &QGstreamerMediaPlayer::mutedChangedHandler);
     connect(gstAudioOutput, &QGstreamerAudioOutput::volumeChanged, this, &QGstreamerMediaPlayer::volumeChangedHandler);
@@ -101,6 +101,7 @@ QGstreamerMediaPlayer::~QGstreamerMediaPlayer()
     playerPipeline.removeMessageFilter(this);
     playerPipeline.setStateSync(GST_STATE_NULL);
     topology.free();
+    delete gstAudioOutput;
 }
 
 qint64 QGstreamerMediaPlayer::position() const
@@ -123,12 +124,12 @@ float QGstreamerMediaPlayer::bufferProgress() const
 
 int QGstreamerMediaPlayer::volume() const
 {
-    return gstAudioOutput->volume();
+    return qRound(gstAudioOutput->volume*100.);
 }
 
 bool QGstreamerMediaPlayer::isMuted() const
 {
-    return gstAudioOutput->isMuted();
+    return gstAudioOutput->muted;
 }
 
 bool QGstreamerMediaPlayer::isSeekable() const
@@ -233,12 +234,21 @@ void QGstreamerMediaPlayer::stopOrEOS(bool eos)
 
 void QGstreamerMediaPlayer::setVolume(int vol)
 {
-    gstAudioOutput->setVolume(vol);
+    float v = vol/100.;
+    if (v == gstAudioOutput->volume)
+        return;
+    gstAudioOutput->volume = v;
+    gstAudioOutput->setVolume(vol/100.);
+    volumeChanged(vol);
 }
 
 void QGstreamerMediaPlayer::setMuted(bool muted)
 {
+    if (muted == gstAudioOutput->muted)
+        return;
+    gstAudioOutput->muted = muted;
     gstAudioOutput->setMuted(muted);
+    mutedChanged(muted);
 }
 
 bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message)
diff --git a/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture.cpp b/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture.cpp
index 80518af3c..d6ee3fdb4 100644
--- a/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture.cpp
+++ b/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture.cpp
@@ -169,14 +169,37 @@ void QGstreamerMediaCapture::setAudioInput(QPlatformAudioInput *input)
 {
     if (gstAudioInput == input)
         return;
+    gstPipeline.setStateSync(GST_STATE_PAUSED);
     if (gstAudioInput) {
+        gstAudioOutput->setPipeline({});
         gstAudioInput = nullptr;
-        setupAudioPipeline();
+        if (gstAudioOutput) {
+            gstAudioOutput->gstElement().setStateSync(GST_STATE_NULL);
+            gstPipeline.remove(gstAudioOutput->gstElement());
+        }
+        if (!gstAudioTee.isNull()) {
+            gstAudioTee.setStateSync(GST_STATE_NULL);
+            gstPipeline.remove(gstAudioTee);
+        }
+        gstAudioTee = {};
     }
     gstAudioInput = static_cast<QGstreamerAudioInput *>(input);
     if (gstAudioInput)
         gstAudioInput->setPipeline(gstPipeline);
-    setupAudioPipeline();
+
+    Q_ASSERT(gstAudioTee.isNull());
+    gstAudioTee = QGstElement("tee", "audiotee");
+    gstAudioTee.set("allow-not-linked", true);
+    gstPipeline.add(gstAudioInput->gstElement(), gstAudioTee);
+    gstAudioInput->gstElement().link(gstAudioTee);
+
+    if (gstAudioOutput) {
+        gstPipeline.add(gstAudioOutput->gstElement());
+        gstAudioOutputPad = gstAudioTee.getRequestPad("src_%u");
+        gstAudioOutputPad.link(gstAudioOutput->gstElement().staticPad("sink"));
+    }
+
+    gstPipeline.setState(GST_STATE_PLAYING);
 }
 
 void QGstreamerMediaCapture::setVideoPreview(QVideoSink *sink)
@@ -184,17 +207,38 @@ void QGstreamerMediaCapture::setVideoPreview(QVideoSink *sink)
     gstVideoOutput->setVideoSink(sink);
 }
 
-QAudioDevice QGstreamerMediaCapture::audioPreview() const
+void QGstreamerMediaCapture::setAudioOutput(QPlatformAudioOutput *output)
 {
-    return gstAudioOutput->audioOutput();
-}
+    if (gstAudioOutput == output)
+        return;
+    gstPipeline.setStateSync(GST_STATE_PAUSED);
 
-bool QGstreamerMediaCapture::setAudioPreview(const QAudioDevice &info)
-{
-    gstAudioOutput->setAudioOutput(info);
-    return true;
+    if (gstAudioOutput) {
+        gstAudioOutput->setPipeline({});
+        gstAudioOutput = nullptr;
+        if (!gstAudioTee.isNull()) {
+            gstAudioOutput->gstElement().setStateSync(GST_STATE_NULL);
+            gstAudioOutputPad.unlinkPeer();
+            gstAudioTee.releaseRequestPad(gstAudioOutputPad);
+            gstAudioOutputPad = {};
+            gstPipeline.remove(gstAudioOutput->gstElement());
+        }
+        setupAudioPipeline();
+    }
+    gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output);
+    if (gstAudioOutput)
+        gstAudioOutput->setPipeline(gstPipeline);
+
+    if (!gstAudioTee.isNull()) {
+        gstPipeline.add(gstAudioOutput->gstElement());
+        gstAudioOutputPad = gstAudioTee.getRequestPad("src_%u");
+        gstAudioOutputPad.link(gstAudioOutput->gstElement().staticPad("sink"));
+    }
+
+    gstPipeline.setState(GST_STATE_PLAYING);
 }
 
+
 QGstPad QGstreamerMediaCapture::getAudioPad() const
 {
     return gstAudioTee.getRequestPad("src_%u");
@@ -219,27 +263,10 @@ void QGstreamerMediaCapture::releaseVideoPad(const QGstPad &pad) const
 
 void QGstreamerMediaCapture::setupAudioPipeline()
 {
-    gstPipeline.setStateSync(GST_STATE_PAUSED);
 
     if (!gstAudioInput) {
-        if (gstAudioOutput)
-            gstPipeline.remove(gstAudioOutput->gstElement());
-        if (!gstAudioTee.isNull())
-            gstPipeline.remove(gstAudioTee);
         return;
     }
-    if (!gstAudioOutput) {
-        gstAudioOutput = new QGstreamerAudioOutput(this);
-        gstAudioOutput->setPipeline(gstPipeline);
-    }
-    gstAudioTee = QGstElement("tee", "audiotee");
-
-    gstPipeline.add(gstAudioInput->gstElement(), gstAudioTee, gstAudioOutput->gstElement());
-    gstAudioInput->gstElement().link(gstAudioTee);
-    auto pad = gstAudioTee.getRequestPad("src_%u");
-    pad.link(gstAudioOutput->gstElement().staticPad("sink"));
-
-    gstPipeline.setStateSync(GST_STATE_PLAYING);
 }
 
 
diff --git a/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture_p.h b/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture_p.h
index 20824443e..bf10054d1 100644
--- a/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture_p.h
+++ b/src/multimedia/platform/gstreamer/mediacapture/qgstreamermediacapture_p.h
@@ -86,8 +86,7 @@ public:
     void setAudioInput(QPlatformAudioInput *input) override;
 
     void setVideoPreview(QVideoSink *sink) override;
-    QAudioDevice audioPreview() const override;
-    bool setAudioPreview(const QAudioDevice &info) override;
+    void setAudioOutput(QPlatformAudioOutput *output) override;
 
     QGstPad getAudioPad() const;
     QGstPad getVideoPad() const;
@@ -109,6 +108,7 @@ private:
     QGstElement gstAudioTee;
     QGstElement gstVideoTee;
 
+    QGstPad gstAudioOutputPad;
     QGstreamerAudioOutput *gstAudioOutput = nullptr;
     QGstreamerVideoOutput *gstVideoOutput = nullptr;
 
diff --git a/src/multimedia/platform/gstreamer/qgstreamerintegration.cpp b/src/multimedia/platform/gstreamer/qgstreamerintegration.cpp
index e834f1141..574fdd079 100644
--- a/src/multimedia/platform/gstreamer/qgstreamerintegration.cpp
+++ b/src/multimedia/platform/gstreamer/qgstreamerintegration.cpp
@@ -116,10 +116,9 @@ QPlatformAudioInput *QGstreamerIntegration::createAudioInput(QAudioInput *q)
     return new QGstreamerAudioInput(q);
 }
 
-QPlatformAudioOutput *QGstreamerIntegration::createAudioOutput(QAudioOutput *)
+QPlatformAudioOutput *QGstreamerIntegration::createAudioOutput(QAudioOutput *q)
 {
-//    return new QGstreamerAudioOutput(q);
-    return nullptr;
+    return new QGstreamerAudioOutput(q);
 }
 
 QT_END_NAMESPACE
diff --git a/src/multimedia/platform/qplatformmediacapture.cpp b/src/multimedia/platform/qplatformmediacapture.cpp
index ee2c53567..b041f69db 100644
--- a/src/multimedia/platform/qplatformmediacapture.cpp
+++ b/src/multimedia/platform/qplatformmediacapture.cpp
@@ -48,10 +48,5 @@ QPlatformMediaCaptureSession::~QPlatformMediaCaptureSession()
 {
 }
 
-QAudioDevice QPlatformMediaCaptureSession::audioPreview() const
-{
-    return QAudioDevice();
-}
-
 QT_END_NAMESPACE
 
diff --git a/src/multimedia/platform/qplatformmediacapture_p.h b/src/multimedia/platform/qplatformmediacapture_p.h
index 46f555d89..222a98686 100644
--- a/src/multimedia/platform/qplatformmediacapture_p.h
+++ b/src/multimedia/platform/qplatformmediacapture_p.h
@@ -61,6 +61,7 @@ class QAudioDevice;
 class QCameraDevice;
 class QVideoSink;
 class QPlatformAudioInput;
+class QPlatformAudioOutput;
 
 class Q_MULTIMEDIA_EXPORT QPlatformMediaCaptureSession : public QObject
 {
@@ -81,8 +82,8 @@ public:
     virtual void setAudioInput(QPlatformAudioInput *input) = 0;
 
     virtual void setVideoPreview(QVideoSink * /*sink*/) {}
-    virtual QAudioDevice audioPreview() const;
-    virtual bool setAudioPreview(const QAudioDevice &) { return true; }
+
+    virtual void setAudioOutput(QPlatformAudioOutput *) {}
 
 Q_SIGNALS:
     void cameraChanged();
diff --git a/src/multimedia/recording/qmediacapturesession.cpp b/src/multimedia/recording/qmediacapturesession.cpp
index 82ed9a558..d07db84d2 100644
--- a/src/multimedia/recording/qmediacapturesession.cpp
+++ b/src/multimedia/recording/qmediacapturesession.cpp
@@ -49,6 +49,7 @@
 #include "qplatformmediaintegration_p.h"
 #include "qplatformmediacapture_p.h"
 #include "qaudioinput.h"
+#include "qaudiooutput.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -58,6 +59,7 @@ public:
     QMediaCaptureSession *q = nullptr;
     QPlatformMediaCaptureSession *captureSession;
     QAudioInput *audioInput = nullptr;
+    QAudioOutput *audioOutput = nullptr;
     QCamera *camera = nullptr;
     QCameraImageCapture *imageCapture = nullptr;
     QMediaEncoder *encoder = nullptr;
@@ -158,7 +160,7 @@ void QMediaCaptureSession::setAudioInput(QAudioInput *device)
     if (d_ptr->audioInput == device)
         return;
     d_ptr->audioInput = device;
-    d_ptr->captureSession->setAudioInput(device->handle());
+    d_ptr->captureSession->setAudioInput(device ? device->handle() : nullptr);
     emit audioInputChanged();
 }
 
@@ -281,6 +283,22 @@ QVideoSink *QMediaCaptureSession::videoSink() const
     return d->videoSink;
 }
 
+void QMediaCaptureSession::setAudioOutput(QAudioOutput *output)
+{
+    Q_D(QMediaCaptureSession);
+    if (d->audioOutput == output)
+        return;
+    d->audioOutput = output;
+    d->captureSession->setAudioOutput(output ? output->handle() : nullptr);
+    emit audioOutputChanged();
+}
+
+QAudioOutput *QMediaCaptureSession::audioOutput() const
+{
+    Q_D(const QMediaCaptureSession);
+    return d->audioOutput;
+}
+
 /*!
     \internal
 */
diff --git a/src/multimedia/recording/qmediacapturesession.h b/src/multimedia/recording/qmediacapturesession.h
index ce5555a93..ee096a147 100644
--- a/src/multimedia/recording/qmediacapturesession.h
+++ b/src/multimedia/recording/qmediacapturesession.h
@@ -47,6 +47,7 @@ QT_BEGIN_NAMESPACE
 
 class QCamera;
 class QAudioInput;
+class QAudioOutput;
 class QCameraDevice;
 class QCameraImageCapture; // ### rename to QMediaImageCapture
 class QMediaEncoder;
@@ -86,6 +87,9 @@ public:
     void setVideoSink(QVideoSink *preview);
     QVideoSink *videoSink() const;
 
+    void setAudioOutput(QAudioOutput *output);
+    QAudioOutput *audioOutput() const;
+
     QPlatformMediaCaptureSession *platformSession() const;
 
 Q_SIGNALS:
@@ -94,6 +98,7 @@ Q_SIGNALS:
     void imageCaptureChanged();
     void encoderChanged();
     void videoOutputChanged();
+    void audioOutputChanged();
 
 private:
     QMediaCaptureSessionPrivate *d_ptr;