Implement audio resampling in QSoundEffect

The resampling in QSoundEffect is needed to align the
audio playback functionality with QMediaPlayer,
otherwise, we get different volume levels when
channels configurations of the source file and the audiodevice
are different.
QMediaPlayer uses ffmpeg resampling, so QSoundEffect is
supposed to use it too.
FFmpeg implementation is to be added in the following commits.

Pick-to: 6.6 6.5
Task-number: QTBUG-118099
Change-Id: I24d0ca394635d53e8fdd4b09f328725dd4f51027
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit f0016c4763361292c3eaf153e184b9b110f44c61)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

4 files changed, 76 insertions, 3 deletions

diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt
index a207ce7dc..7d231998b 100644
--- a/src/multimedia/CMakeLists.txt
+++ b/src/multimedia/CMakeLists.txt
@@ -40,6 +40,7 @@ qt_internal_add_module(Multimedia
         platform/qplatformaudiodecoder.cpp platform/qplatformaudiodecoder_p.h
         platform/qplatformaudioinput_p.h
         platform/qplatformaudiooutput_p.h
+        platform/qplatformaudioresampler_p.h
         platform/qplatformcamera.cpp platform/qplatformcamera_p.h
         platform/qplatformvideosource.cpp platform/qplatformvideosource_p.h
         platform/qplatformsurfacecapture.cpp platform/qplatformsurfacecapture_p.h
diff --git a/src/multimedia/audio/qsoundeffect.cpp b/src/multimedia/audio/qsoundeffect.cpp
index bff9dc093..ee639b41d 100644
--- a/src/multimedia/audio/qsoundeffect.cpp
+++ b/src/multimedia/audio/qsoundeffect.cpp
@@ -7,8 +7,11 @@
 #include "qaudiodevice.h"
 #include "qaudiosink.h"
 #include "qmediadevices.h"
+#include "qaudiobuffer.h"
 #include <QtCore/qloggingcategory.h>
 #include <private/qplatformmediadevices_p.h>
+#include <private/qplatformmediaintegration_p.h>
+#include <private/qplatformaudioresampler_p.h>
 
 static Q_LOGGING_CATEGORY(qLcSoundEffect, "qt.multimedia.soundeffect")
 
@@ -81,6 +84,7 @@ public:
     QSoundEffect::Status m_status = QSoundEffect::Null;
     std::unique_ptr<QAudioSink, AudioSinkDeleter> m_audioSink;
     std::unique_ptr<QSample, SampleDeleter> m_sample;
+    QAudioBuffer m_audioBuffer;
     bool m_muted = false;
     float m_volume = 1.0;
     bool m_sampleReady = false;
@@ -107,7 +111,35 @@ void QSoundEffectPrivate::sampleReady()
     disconnect(m_sample.get(), &QSample::error, this, &QSoundEffectPrivate::decoderError);
     disconnect(m_sample.get(), &QSample::ready, this, &QSoundEffectPrivate::sampleReady);
     if (!m_audioSink) {
-        m_audioSink.reset(new QAudioSink(m_audioDevice, m_sample->format()));
+        const auto audioDevice =
+                m_audioDevice.isNull() ? QMediaDevices::defaultAudioOutput() : m_audioDevice;
+        const auto &sampleFormat = m_sample->format();
+        const auto sampleChannelConfig =
+                sampleFormat.channelConfig() == QAudioFormat::ChannelConfigUnknown
+                ? QAudioFormat::defaultChannelConfigForChannelCount(sampleFormat.channelCount())
+                : sampleFormat.channelConfig();
+
+        if (sampleChannelConfig != audioDevice.channelConfiguration()
+            && audioDevice.channelConfiguration() != QAudioFormat::ChannelConfigUnknown) {
+            qCDebug(qLcSoundEffect) << "Create resampler for channels mapping: config"
+                                    << sampleFormat.channelConfig() << "=> config"
+                                    << audioDevice.channelConfiguration();
+            auto outputFormat = sampleFormat;
+            outputFormat.setChannelConfig(audioDevice.channelConfiguration());
+            if (auto maybeResampler = QPlatformMediaIntegration::instance()->createAudioResampler(
+                        m_sample->format(), outputFormat)) {
+                std::unique_ptr<QPlatformAudioResampler> resampler(maybeResampler.value());
+                m_audioBuffer =
+                        resampler->resample(m_sample->data().constData(), m_sample->data().size());
+            } else {
+                qCDebug(qLcSoundEffect) << "Cannot create resampler for channels mapping";
+            }
+        }
+
+        if (!m_audioBuffer.isValid())
+            m_audioBuffer = QAudioBuffer(m_sample->data(), m_sample->format());
+
+        m_audioSink.reset(new QAudioSink(audioDevice, m_audioBuffer.format()));
         connect(m_audioSink.get(), &QAudioSink::stateChanged, this, &QSoundEffectPrivate::stateChanged);
         if (!m_muted)
             m_audioSink->setVolume(m_volume);
@@ -151,8 +183,8 @@ qint64 QSoundEffectPrivate::readData(char *data, qint64 len)
 
     qint64 bytesWritten = 0;
 
-    const int   sampleSize = m_sample->data().size();
-    const char* sampleData = m_sample->data().constData();
+    const int sampleSize = m_audioBuffer.byteCount();
+    const char *sampleData = m_audioBuffer.constData<char>();
 
     while (len && m_runningCount) {
         int toWrite = qMin(sampleSize - m_offset, len);
diff --git a/src/multimedia/platform/qplatformaudioresampler_p.h b/src/multimedia/platform/qplatformaudioresampler_p.h
new file mode 100644
index 000000000..fd8edc2be
--- /dev/null
+++ b/src/multimedia/platform/qplatformaudioresampler_p.h
@@ -0,0 +1,33 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QPLATFORMAUDIORESAMPLER_P_H
+#define QPLATFORMAUDIORESAMPLER_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <private/qtmultimediaglobal_p.h>
+#include <qaudiobuffer.h>
+
+QT_BEGIN_NAMESPACE
+
+class QPlatformAudioResampler
+{
+public:
+    virtual ~QPlatformAudioResampler() = default;
+
+    virtual QAudioBuffer resample(const char *data, size_t size) = 0;
+};
+
+QT_END_NAMESPACE
+
+#endif // QPLATFORMAUDIORESAMPLER_P_H
diff --git a/src/multimedia/platform/qplatformmediaintegration_p.h b/src/multimedia/platform/qplatformmediaintegration_p.h
index cc40627d0..21891b866 100644
--- a/src/multimedia/platform/qplatformmediaintegration_p.h
+++ b/src/multimedia/platform/qplatformmediaintegration_p.h
@@ -37,6 +37,7 @@ class QPlatformMediaDevices;
 class QPlatformMediaCaptureSession;
 class QPlatformMediaPlayer;
 class QPlatformAudioDecoder;
+class QPlatformAudioResampler;
 class QPlatformCamera;
 class QPlatformSurfaceCapture;
 class QPlatformMediaRecorder;
@@ -69,6 +70,12 @@ public:
     virtual QPlatformSurfaceCapture *createWindowCapture(QWindowCapture *) { return nullptr; }
 
     virtual QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *) { return notAvailable; }
+    virtual QMaybe<QPlatformAudioResampler *>
+    createAudioResampler(const QAudioFormat & /*inputFormat*/,
+                         const QAudioFormat & /*outputFormat*/)
+    {
+        return notAvailable;
+    }
     virtual QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() { return notAvailable; }
     virtual QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *) { return notAvailable; }
     virtual QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *) { return notAvailable; }