macOS/iOS: Support setting audioOutput device to the capture session

Change-Id: I4bdc5c0120a1ab26906c05d854fb66d71fab4efe
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
(cherry picked from commit f63d082503138ba8fa40ec05fdc1702758cc34d9)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

8 files changed, 84 insertions, 3 deletions

diff --git a/src/multimedia/platform/darwin/camera/avfcameraservice.mm b/src/multimedia/platform/darwin/camera/avfcameraservice.mm
index b669eacc1..de73542bc 100644
--- a/src/multimedia/platform/darwin/camera/avfcameraservice.mm
+++ b/src/multimedia/platform/darwin/camera/avfcameraservice.mm
@@ -49,7 +49,9 @@
 #include "avfmediaencoder_p.h"
 #include <qmediadevices.h>
 #include <private/qplatformaudioinput_p.h>
+#include <private/qplatformaudiooutput_p.h>
 #include <qaudioinput.h>
+#include <qaudiooutput.h>
 
 QT_USE_NAMESPACE
 
@@ -120,6 +122,8 @@ void AVFCameraService::setMediaEncoder(QPlatformMediaEncoder *encoder)
     m_encoder = control;
     if (m_encoder)
         m_encoder->setCaptureSession(this);
+
+    audioOutputChanged();
     emit encoderChanged();
 }
 
@@ -139,6 +143,22 @@ void AVFCameraService::setAudioInput(QPlatformAudioInput *input)
     audioInputChanged();
 }
 
+void AVFCameraService::setAudioOutput(QPlatformAudioOutput *output)
+{
+    if (m_audioOutput == output)
+        return;
+    if (m_audioOutput)
+        m_audioOutput->q->disconnect(this);
+
+    m_audioOutput = output;
+
+    if (m_audioOutput) {
+        connect(m_audioOutput->q, &QAudioOutput::destroyed, this, &AVFCameraService::audioOutputDestroyed);
+        connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this, &AVFCameraService::audioOutputChanged);
+    }
+    audioOutputChanged();
+}
+
 void AVFCameraService::audioInputChanged()
 {
     m_audioCaptureDevice = nullptr;
@@ -154,6 +174,14 @@ void AVFCameraService::audioInputChanged()
     m_session->updateAudioInput();
 }
 
+void AVFCameraService::audioOutputChanged()
+{
+    if (!m_audioOutput)
+        return;
+    if (m_encoder)
+        m_encoder->onAudioOutputChanged();
+}
+
 void AVFCameraService::setVideoPreview(QVideoSink *sink)
 {
     if (m_session)
diff --git a/src/multimedia/platform/darwin/camera/avfcameraservice_p.h b/src/multimedia/platform/darwin/camera/avfcameraservice_p.h
index 5e641cb6a..7e9133703 100644
--- a/src/multimedia/platform/darwin/camera/avfcameraservice_p.h
+++ b/src/multimedia/platform/darwin/camera/avfcameraservice_p.h
@@ -82,6 +82,7 @@ public:
     void setMediaEncoder(QPlatformMediaEncoder *encoder) override;
 
     void setAudioInput(QPlatformAudioInput *) override;
+    void setAudioOutput(QPlatformAudioOutput *) override;
 
     void setVideoPreview(QVideoSink *sink) override;
 
@@ -91,11 +92,18 @@ public:
     AVFImageCapture *avfImageCaptureControl() const { return m_imageCaptureControl; }
     AVCaptureDevice *audioCaptureDevice() const { return m_audioCaptureDevice; }
 
+    QPlatformAudioOutput *audioOutput() { return m_audioOutput; }
+
 public Q_SLOTS:
     void audioInputDestroyed() { setAudioInput(nullptr); }
     void audioInputChanged();
+    void audioOutputDestroyed() { setAudioOutput(nullptr); }
+    void audioOutputChanged();
+
 private:
     QPlatformAudioInput *m_audioInput = nullptr;
+    QPlatformAudioOutput *m_audioOutput = nullptr;
+
     AVCaptureDevice *m_audioCaptureDevice = nullptr;
 
     AVFCameraSession *m_session = nullptr;
diff --git a/src/multimedia/platform/darwin/camera/avfcamerasession.mm b/src/multimedia/platform/darwin/camera/avfcamerasession.mm
index 7f2ca7b5b..ab3d7c0f8 100644
--- a/src/multimedia/platform/darwin/camera/avfcamerasession.mm
+++ b/src/multimedia/platform/darwin/camera/avfcamerasession.mm
@@ -201,7 +201,7 @@ void AVFCameraSession::setVideoOutput(AVFCameraRenderer *output)
         output->configureAVCaptureSession(this);
 }
 
-void AVFCameraSession::setAudioOutput()
+void AVFCameraSession::addAudioCapture()
 {
     if (m_audioOutput) {
         [m_captureSession removeOutput:m_audioOutput];
@@ -252,7 +252,7 @@ void AVFCameraSession::setActive(bool active)
 
         if (m_service->audioCaptureDevice()) {
             attachAudioInputDevice();
-            setAudioOutput();
+            addAudioCapture();
         }
         if (!m_activeCameraDevice.isNull()) {
             attachVideoInputDevice();
diff --git a/src/multimedia/platform/darwin/camera/avfcamerasession_p.h b/src/multimedia/platform/darwin/camera/avfcamerasession_p.h
index fee807c21..15c809b30 100644
--- a/src/multimedia/platform/darwin/camera/avfcamerasession_p.h
+++ b/src/multimedia/platform/darwin/camera/avfcamerasession_p.h
@@ -109,7 +109,7 @@ Q_SIGNALS:
 
 private:
     void setVideoOutput(AVFCameraRenderer *output);
-    void setAudioOutput();
+    void addAudioCapture();
     AVCaptureDevice *createVideoCaptureDevice();
     void attachVideoInputDevice();
     void attachAudioInputDevice();
diff --git a/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm b/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm
index 0523a7931..394825fe1 100644
--- a/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm
+++ b/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm
@@ -99,6 +99,9 @@ using AVFAtomicInt64 = QAtomicInteger<qint64>;
 
     AVFScopedPointer<AVAssetWriter> m_assetWriter;
 
+    AVFScopedPointer<AVSampleBufferRenderSynchronizer> m_bufferSynchronizer;
+    AVFScopedPointer<AVSampleBufferAudioRenderer> m_audioRenderer;
+
     AVFMediaEncoder *m_delegate;
 
     bool m_setStartTime;
@@ -342,6 +345,13 @@ using AVFAtomicInt64 = QAtomicInteger<qint64>;
         }
     }
 
+    if (m_audioRenderer && m_bufferSynchronizer) {
+        [m_audioRenderer enqueueSampleBuffer:sampleBuffer];
+        if (m_bufferSynchronizer.data().rate == 0)
+            [m_bufferSynchronizer setRate:1
+             time:CMSampleBufferGetPresentationTimeStamp(sampleBuffer)];
+    }
+
     CFRelease(sampleBuffer);
 }
 
@@ -479,4 +489,24 @@ using AVFAtomicInt64 = QAtomicInteger<qint64>;
     return m_durationInMs.loadAcquire();
 }
 
+- (void)updateAudioOutput:(NSString *)deviceId
+{
+    if (!m_bufferSynchronizer)
+        m_bufferSynchronizer.reset([[AVSampleBufferRenderSynchronizer alloc] init]);
+    if (!m_audioRenderer)
+        m_audioRenderer.reset([[AVSampleBufferAudioRenderer alloc] init]);
+
+    if (![m_bufferSynchronizer.data().renderers containsObject:m_audioRenderer.data()])
+        [m_bufferSynchronizer addRenderer:m_audioRenderer];
+
+#ifdef Q_OS_MACOS
+    m_audioRenderer.data().audioOutputDeviceUniqueID = deviceId;
+#endif
+
+    if (!deviceId) {
+        m_bufferSynchronizer.data().rate = 0.0;
+        [m_audioRenderer flush];
+    }
+}
+
 @end
diff --git a/src/multimedia/platform/darwin/camera/avfmediaassetwriter_p.h b/src/multimedia/platform/darwin/camera/avfmediaassetwriter_p.h
index 92ef9f874..83e6b55b5 100644
--- a/src/multimedia/platform/darwin/camera/avfmediaassetwriter_p.h
+++ b/src/multimedia/platform/darwin/camera/avfmediaassetwriter_p.h
@@ -81,6 +81,8 @@ QT_END_NAMESPACE
 - (void)abort;
 - (qint64)durationInMs;
 
+- (void)updateAudioOutput:(NSString *)deviceId;
+
 @end
 
 #endif // AVFMEDIAASSETWRITER_H
diff --git a/src/multimedia/platform/darwin/camera/avfmediaencoder.mm b/src/multimedia/platform/darwin/camera/avfmediaencoder.mm
index b45f48335..bb615bf33 100644
--- a/src/multimedia/platform/darwin/camera/avfmediaencoder.mm
+++ b/src/multimedia/platform/darwin/camera/avfmediaencoder.mm
@@ -50,6 +50,7 @@
 #include "qmediadevices.h"
 #include "private/qmediarecorder_p.h"
 #include "private/qdarwinformatsinfo_p.h"
+#include "private/qplatformaudiooutput_p.h"
 
 #include <QtCore/qmath.h>
 #include <QtCore/qdebug.h>
@@ -629,4 +630,15 @@ void AVFMediaEncoder::stopWriter()
     }
 }
 
+void AVFMediaEncoder::onAudioOutputChanged()
+{
+    QPlatformAudioOutput *audioOutput = m_service ? m_service->audioOutput()
+                                                  : nullptr;
+    NSString *deviceId = nil;
+    if (audioOutput)
+        deviceId = QString::fromUtf8(audioOutput->device.id()).toNSString();
+
+    [m_writer updateAudioOutput:deviceId];
+}
+
 #include "moc_avfmediaencoder_p.cpp"
diff --git a/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h b/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h
index 7c57768b0..b9f4c0a23 100644
--- a/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h
+++ b/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h
@@ -100,6 +100,7 @@ public:
 
 public Q_SLOTS:
     void setState(QMediaRecorder::RecorderState state) override;
+    void onAudioOutputChanged();
 
 private: