Implement audio resampler in the ffmpeg plugin

QFFmpeg::Resampler has been renamed and reworked for inheritance from
QPlatformAudioResampler.
The main purpose of the set of commits is to align the sound played by
QMediaPlayer and QSoundEffect. The classes have different volume
levels and channel mixing factors if we use ffmpeg resampling in one
and doesn't use in another one.

Pick-to: 6.6 6.5
Task-number: QTBUG-118099
Change-Id: I01cb909de756d5e7cd0f1c40e749f186efd2f649
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
(cherry picked from commit 5d1c4ae289a1e22662c7c7c192481b386a21a84c)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

7 files changed, 62 insertions, 30 deletions

diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp
index 9f8ea23a1..88caac941 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp
@@ -186,7 +186,7 @@ void AudioRenderer::initResempler(const Codec *codec)
     auto resamplerFormat = m_format;
     resamplerFormat.setSampleRate(
             qRound(m_format.sampleRate() / playbackRate() * sampleRateFactor()));
-    m_resampler = std::make_unique<Resampler>(codec, resamplerFormat);
+    m_resampler = std::make_unique<QFFmpegResampler>(codec, resamplerFormat);
 }
 
 void AudioRenderer::freeOutput()
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h
index 02bb2a46e..196cd4fd0 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h
@@ -22,10 +22,7 @@ QT_BEGIN_NAMESPACE
 
 class QAudioOutput;
 class QAudioSink;
-
-namespace QFFmpeg {
-class Resampler;
-};
+class QFFmpegResampler;
 
 namespace QFFmpeg {
 
@@ -105,7 +102,7 @@ private:
     std::unique_ptr<QAudioSink> m_sink;
     AudioTimings m_timings;
     BufferLoadingInfo m_bufferLoadingInfo;
-    std::unique_ptr<Resampler> m_resampler;
+    std::unique_ptr<QFFmpegResampler> m_resampler;
     QAudioFormat m_format;
 
     BufferedDataWithOffset m_bufferedData;
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp
index fee9a8821..7482a3390 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp
@@ -28,7 +28,7 @@ public:
             return {};
 
         if (!m_resampler)
-            m_resampler = std::make_unique<Resampler>(frame.codec(), m_format);
+            m_resampler = std::make_unique<QFFmpegResampler>(frame.codec(), m_format);
 
         emit newAudioBuffer(m_resampler->resample(frame.avFrame()));
 
@@ -40,7 +40,7 @@ signals:
 
 private:
     QAudioFormat m_format;
-    std::unique_ptr<Resampler> m_resampler;
+    std::unique_ptr<QFFmpegResampler> m_resampler;
 };
 
 class AudioDecoder : public PlaybackEngine
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
index b2c2081bb..7abded14c 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
@@ -12,6 +12,7 @@
 #include "qffmpegimagecapture_p.h"
 #include "qffmpegaudioinput_p.h"
 #include "qffmpegaudiodecoder_p.h"
+#include "qffmpegresampler_p.h"
 #include "qffmpegsymbolsresolve_p.h"
 #include "qgrabwindowsurfacecapture_p.h"
 
@@ -189,6 +190,11 @@ QMaybe<QPlatformAudioDecoder *> QFFmpegMediaIntegration::createAudioDecoder(QAud
     return new QFFmpegAudioDecoder(decoder);
 }
 
+QMaybe<QPlatformAudioResampler *> QFFmpegMediaIntegration::createAudioResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat)
+{
+    return new QFFmpegResampler(inputFormat, outputFormat);
+}
+
 QMaybe<QPlatformMediaCaptureSession *> QFFmpegMediaIntegration::createCaptureSession()
 {
     return new QFFmpegMediaCaptureSession();
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h
index 5e174f327..0e773c585 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h
@@ -32,6 +32,7 @@ public:
     }
 
     QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *decoder) override;
+    QMaybe<QPlatformAudioResampler *> createAudioResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat) override;
     QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override;
     QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override;
     QMaybe<QPlatformCamera *> createCamera(QCamera *) override;
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp b/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp
index c5c277213..60482eb96 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp
@@ -9,10 +9,19 @@ static Q_LOGGING_CATEGORY(qLcResampler, "qt.multimedia.ffmpeg.resampler")
 
 QT_BEGIN_NAMESPACE
 
-namespace QFFmpeg
+using namespace QFFmpeg;
+
+QFFmpegResampler::QFFmpegResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat) :
+    m_inputFormat(inputFormat), m_outputFormat(outputFormat)
 {
+    Q_ASSERT(inputFormat.isValid());
+    Q_ASSERT(outputFormat.isValid());
+
+    m_resampler = createResampleContext(ResampleAudioFormat(m_inputFormat),
+                                        ResampleAudioFormat(m_outputFormat));
+}
 
-Resampler::Resampler(const Codec *codec, const QAudioFormat &outputFormat)
+QFFmpegResampler::QFFmpegResampler(const Codec* codec, const QAudioFormat &outputFormat)
     : m_outputFormat(outputFormat)
 {
     Q_ASSERT(codec);
@@ -28,31 +37,44 @@ Resampler::Resampler(const Codec *codec, const QAudioFormat &outputFormat)
                                         ResampleAudioFormat(m_outputFormat));
 }
 
-Resampler::~Resampler() = default;
+QFFmpegResampler::~QFFmpegResampler() = default;
+
+QAudioBuffer QFFmpegResampler::resample(const char* data, size_t size)
+{
+    if (!m_inputFormat.isValid())
+        return {};
+
+    return resample(reinterpret_cast<const uint8_t **>(&data),
+                    m_inputFormat.framesForBytes(static_cast<qint32>(size)));
+}
+
+QAudioBuffer QFFmpegResampler::resample(const AVFrame *frame)
+{
+    return resample(const_cast<const uint8_t **>(frame->extended_data), frame->nb_samples);
+}
 
-QAudioBuffer Resampler::resample(const AVFrame *frame)
+QAudioBuffer QFFmpegResampler::resample(const uint8_t **inputData, int inputSamplesCount)
 {
-    const int maxOutSamples = adjustMaxOutSamples(frame);
+    const int maxOutSamples = adjustMaxOutSamples(inputSamplesCount);
 
     QByteArray samples(m_outputFormat.bytesForFrames(maxOutSamples), Qt::Uninitialized);
-    auto **in = const_cast<const uint8_t **>(frame->extended_data);
     auto *out = reinterpret_cast<uint8_t *>(samples.data());
     const int outSamples =
-            swr_convert(m_resampler.get(), &out, maxOutSamples, in, frame->nb_samples);
+            swr_convert(m_resampler.get(), &out, maxOutSamples, inputData, inputSamplesCount);
 
     samples.resize(m_outputFormat.bytesForFrames(outSamples));
 
     qint64 startTime = m_outputFormat.durationForFrames(m_samplesProcessed);
     m_samplesProcessed += outSamples;
 
-    qCDebug(qLcResampler) << "    new frame" << startTime << "in_samples" << frame->nb_samples
+    qCDebug(qLcResampler) << "    new frame" << startTime << "in_samples" << inputSamplesCount
                           << outSamples << maxOutSamples;
     return QAudioBuffer(samples, m_outputFormat, startTime);
 }
 
-int Resampler::adjustMaxOutSamples(const AVFrame *frame)
+int QFFmpegResampler::adjustMaxOutSamples(int inputSamplesCount)
 {
-    int maxOutSamples = swr_get_out_samples(m_resampler.get(), frame->nb_samples);
+    int maxOutSamples = swr_get_out_samples(m_resampler.get(), inputSamplesCount);
 
     const auto remainingCompensationDistance = m_endCompensationSample - m_samplesProcessed;
 
@@ -64,13 +86,13 @@ int Resampler::adjustMaxOutSamples(const AVFrame *frame)
         // however it's not significant for our logic, in fact.
         // TODO: probably, it will need some improvements
         setSampleCompensation(0, 0);
-        maxOutSamples = swr_get_out_samples(m_resampler.get(), frame->nb_samples);
+        maxOutSamples = swr_get_out_samples(m_resampler.get(), inputSamplesCount);
     }
 
     return maxOutSamples;
 }
 
-void Resampler::setSampleCompensation(qint32 delta, quint32 distance)
+void QFFmpegResampler::setSampleCompensation(qint32 delta, quint32 distance)
 {
     const int res = swr_set_compensation(m_resampler.get(), delta, static_cast<int>(distance));
     if (res < 0)
@@ -81,10 +103,9 @@ void Resampler::setSampleCompensation(qint32 delta, quint32 distance)
     }
 }
 
-qint32 Resampler::activeSampleCompensationDelta() const
+qint32 QFFmpegResampler::activeSampleCompensationDelta() const
 {
     return m_samplesProcessed < m_endCompensationSample ? m_sampleCompensationDelta : 0;
 }
-}
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h b/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h
index 55802a17d..5109ecf11 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h
@@ -16,38 +16,45 @@
 
 #include "qaudiobuffer.h"
 #include "qffmpeg_p.h"
+#include "private/qplatformaudioresampler_p.h"
 
 QT_BEGIN_NAMESPACE
 
 namespace QFFmpeg
 {
-
 class Codec;
+}
 
-class Resampler
+class QFFmpegResampler : public QPlatformAudioResampler
 {
 public:
-    Resampler(const Codec *codec, const QAudioFormat &outputFormat);
-    ~Resampler();
+    QFFmpegResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat);
+    QFFmpegResampler(const QFFmpeg::Codec* codec, const QAudioFormat &outputFormat);
+
+    ~QFFmpegResampler() override;
+
+    QAudioBuffer resample(const char* data, size_t size) override;
 
     QAudioBuffer resample(const AVFrame *frame);
+
     qint64 samplesProcessed() const { return m_samplesProcessed; }
     void setSampleCompensation(qint32 delta, quint32 distance);
     qint32 activeSampleCompensationDelta() const;
 
 private:
-    int adjustMaxOutSamples(const AVFrame *frame);
+    int adjustMaxOutSamples(int inputSamplesCount);
+
+    QAudioBuffer resample(const uint8_t **inputData, int inputSamplesCount);
 
 private:
+    QAudioFormat m_inputFormat;
     QAudioFormat m_outputFormat;
-    SwrContextUPtr m_resampler;
+    QFFmpeg::SwrContextUPtr m_resampler;
     qint64 m_samplesProcessed = 0;
     qint64 m_endCompensationSample = std::numeric_limits<qint64>::min();
     qint32 m_sampleCompensationDelta = 0;
 };
 
-}
-
 QT_END_NAMESPACE
 
 #endif