diff options
author | Jøger Hansegård <joger.hansegard@qt.io> | 2024-04-01 16:14:32 +0200 |
---|---|---|
committer | Qt Cherry-pick Bot <cherrypick_bot@qt-project.org> | 2024-04-02 02:15:06 +0000 |
commit | 3e1bd6cf8e59a4802a4261b984a4c5627a7e0ee6 (patch) | |
tree | 175fa76576f9a05bb8937e412deca0931ee5aa1e | |
parent | 0241788e99ff236bf52d07ee8a83c4f60b7c9afc (diff) |
Move FFmpeg AudioEncoder class to its own files
This helps readability because the qffmpegrecordingengine_p.h file is
quite big.
Task-number: QTBUG-121792
Pick-to: 6.5
Change-Id: Icc693073e16239076a250cb1ad0de21562c59375
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
(cherry picked from commit f232089960ce56d1abe1524e0d63f175c78a71e2)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit 1bda134e6a0409dcd2e4952eac5f3a41d99cc706)
5 files changed, 280 insertions, 236 deletions
diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt index c6c1fd37f..c94fa3e1f 100644 --- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt +++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt @@ -54,6 +54,8 @@ qt_internal_add_plugin(QFFmpegMediaPlugin playbackengine/qffmpegframe_p.h playbackengine/qffmpegpositionwithoffset_p.h + recordingengine/qffmpegaudioencoder_p.h + recordingengine/qffmpegaudioencoder.cpp recordingengine/qffmpegaudioencoderutils_p.h recordingengine/qffmpegaudioencoderutils.cpp recordingengine/qffmpegencoderoptions_p.h diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder.cpp new file mode 100644 index 000000000..1d825844e --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder.cpp @@ -0,0 +1,221 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegaudioencoder_p.h" +#include "qffmpegaudioencoderutils_p.h" +#include "qffmpegaudioinput_p.h" +#include "qffmpegencoderoptions_p.h" +#include "qffmpegmuxer_p.h" +#include "qffmpegrecordingengine_p.h" +#include "qffmpegmediaformatinfo_p.h" +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +static Q_LOGGING_CATEGORY(qLcFFmpegAudioEncoder, "qt.multimedia.ffmpeg.audioencoder"); + +AudioEncoder::AudioEncoder(RecordingEngine *encoder, QFFmpegAudioInput *input, + const QMediaEncoderSettings &settings) + : EncoderThread(encoder), m_input(input), m_settings(settings) +{ + setObjectName(QLatin1String("AudioEncoder")); + qCDebug(qLcFFmpegAudioEncoder) << "AudioEncoder" << settings.audioCodec(); + + m_format = input->device.preferredFormat(); + auto codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(settings.audioCodec()); + Q_ASSERT(avformat_query_codec(encoder->avFormatContext()->oformat, codecID, + FF_COMPLIANCE_NORMAL)); + + const AVAudioFormat requestedAudioFormat(m_format); + + m_avCodec = QFFmpeg::findAVEncoder(codecID, {}, requestedAudioFormat.sampleFormat); + + if (!m_avCodec) + m_avCodec = QFFmpeg::findAVEncoder(codecID); + + qCDebug(qLcFFmpegAudioEncoder) << "found audio codec" << m_avCodec->name; + + Q_ASSERT(m_avCodec); + + m_stream = avformat_new_stream(encoder->avFormatContext(), nullptr); + m_stream->id = encoder->avFormatContext()->nb_streams - 1; + m_stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; + m_stream->codecpar->codec_id = codecID; +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + m_stream->codecpar->channel_layout = + adjustChannelLayout(m_avCodec->channel_layouts, requestedAudioFormat.channelLayoutMask); + m_stream->codecpar->channels = qPopulationCount(m_stream->codecpar->channel_layout); +#else + m_stream->codecpar->ch_layout = + adjustChannelLayout(m_avCodec->ch_layouts, requestedAudioFormat.channelLayout); +#endif + const auto sampleRate = + adjustSampleRate(m_avCodec->supported_samplerates, requestedAudioFormat.sampleRate); + + m_stream->codecpar->sample_rate = sampleRate; + m_stream->codecpar->frame_size = 1024; + m_stream->codecpar->format = + adjustSampleFormat(m_avCodec->sample_fmts, requestedAudioFormat.sampleFormat); + + m_stream->time_base = AVRational{ 1, sampleRate }; + + qCDebug(qLcFFmpegAudioEncoder) << "set stream time_base" << m_stream->time_base.num << "/" + << m_stream->time_base.den; +} + +void AudioEncoder::open() +{ + m_codecContext.reset(avcodec_alloc_context3(m_avCodec)); + + if (m_stream->time_base.num != 1 || m_stream->time_base.den != m_format.sampleRate()) { + qCDebug(qLcFFmpegAudioEncoder) << "Most likely, av_format_write_header changed time base from" + << 1 << "/" << m_format.sampleRate() << "to" + << m_stream->time_base; + } + + m_codecContext->time_base = m_stream->time_base; + + avcodec_parameters_to_context(m_codecContext.get(), m_stream->codecpar); + + AVDictionaryHolder opts; + applyAudioEncoderOptions(m_settings, m_avCodec->name, m_codecContext.get(), opts); + applyExperimentalCodecOptions(m_avCodec, opts); + + int res = avcodec_open2(m_codecContext.get(), m_avCodec, opts); + qCDebug(qLcFFmpegAudioEncoder) << "audio codec opened" << res; + qCDebug(qLcFFmpegAudioEncoder) << "audio codec params: fmt=" << m_codecContext->sample_fmt + << "rate=" << m_codecContext->sample_rate; + + const AVAudioFormat requestedAudioFormat(m_format); + const AVAudioFormat codecAudioFormat(m_codecContext.get()); + + if (requestedAudioFormat != codecAudioFormat) + m_resampler = createResampleContext(requestedAudioFormat, codecAudioFormat); +} + +void AudioEncoder::addBuffer(const QAudioBuffer &buffer) +{ + QMutexLocker locker(&m_queueMutex); + if (!m_paused.loadRelaxed()) { + m_audioBufferQueue.push(buffer); + locker.unlock(); + dataReady(); + } +} + +QAudioBuffer AudioEncoder::takeBuffer() +{ + QMutexLocker locker(&m_queueMutex); + return dequeueIfPossible(m_audioBufferQueue); +} + +void AudioEncoder::init() +{ + open(); + if (m_input) { + m_input->setFrameSize(m_codecContext->frame_size); + } + qCDebug(qLcFFmpegAudioEncoder) << "AudioEncoder::init started audio device thread."; +} + +void AudioEncoder::cleanup() +{ + while (!m_audioBufferQueue.empty()) + processOne(); + while (avcodec_send_frame(m_codecContext.get(), nullptr) == AVERROR(EAGAIN)) + retrievePackets(); + retrievePackets(); +} + +bool AudioEncoder::hasData() const +{ + QMutexLocker locker(&m_queueMutex); + return !m_audioBufferQueue.empty(); +} + +void AudioEncoder::retrievePackets() +{ + while (1) { + AVPacketUPtr packet(av_packet_alloc()); + int ret = avcodec_receive_packet(m_codecContext.get(), packet.get()); + if (ret < 0) { + if (ret != AVERROR(EOF)) + break; + if (ret != AVERROR(EAGAIN)) { + char errStr[1024]; + av_strerror(ret, errStr, 1024); + qCDebug(qLcFFmpegAudioEncoder) << "receive packet" << ret << errStr; + } + break; + } + + // qCDebug(qLcFFmpegEncoder) << "writing audio packet" << packet->size << packet->pts << + // packet->dts; + packet->stream_index = m_stream->id; + m_encoder->m_muxer->addPacket(std::move(packet)); + } +} + +void AudioEncoder::processOne() +{ + QAudioBuffer buffer = takeBuffer(); + if (!buffer.isValid()) + return; + + if (buffer.format() != m_format) { + // should we recreate recreate resampler here? + qWarning() << "Get invalid audio format:" << buffer.format() << ", expected:" << m_format; + return; + } + + // qCDebug(qLcFFmpegEncoder) << "new audio buffer" << buffer.byteCount() << buffer.format() + // << buffer.frameCount() << codec->frame_size; + retrievePackets(); + + auto frame = makeAVFrame(); + frame->format = m_codecContext->sample_fmt; +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + frame->channel_layout = m_codecContext->channel_layout; + frame->channels = m_codecContext->channels; +#else + frame->ch_layout = m_codecContext->ch_layout; +#endif + frame->sample_rate = m_codecContext->sample_rate; + frame->nb_samples = buffer.frameCount(); + if (frame->nb_samples) + av_frame_get_buffer(frame.get(), 0); + + if (m_resampler) { + const uint8_t *data = buffer.constData<uint8_t>(); + swr_convert(m_resampler.get(), frame->extended_data, frame->nb_samples, &data, + frame->nb_samples); + } else { + memcpy(frame->buf[0]->data, buffer.constData<uint8_t>(), buffer.byteCount()); + } + + const auto &timeBase = m_stream->time_base; + const auto pts = timeBase.den && timeBase.num + ? timeBase.den * m_samplesWritten / (m_codecContext->sample_rate * timeBase.num) + : m_samplesWritten; + setAVFrameTime(*frame, pts, timeBase); + m_samplesWritten += buffer.frameCount(); + + qint64 time = m_format.durationForFrames(m_samplesWritten); + m_encoder->newTimeStamp(time / 1000); + + // qCDebug(qLcFFmpegEncoder) << "sending audio frame" << buffer.byteCount() << frame->pts << + // ((double)buffer.frameCount()/frame->sample_rate); + + int ret = avcodec_send_frame(m_codecContext.get(), frame.get()); + if (ret < 0) { + char errStr[1024]; + av_strerror(ret, errStr, 1024); + // qCDebug(qLcFFmpegEncoder) << "error sending frame" << ret << errStr; + } +} + + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder_p.h new file mode 100644 index 000000000..d1de9ca29 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder_p.h @@ -0,0 +1,56 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGAUDIOENCODER_P_H +#define QFFMPEGAUDIOENCODER_P_H + +#include "qffmpeg_p.h" +#include "qffmpegrecordingengine_p.h" +#include <qaudiobuffer.h> +#include <QtCore/QtGlobal> +#include <queue> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class AudioEncoder : public EncoderThread +{ +public: + AudioEncoder(RecordingEngine *encoder, QFFmpegAudioInput *input, + const QMediaEncoderSettings &settings); + + void open(); + void addBuffer(const QAudioBuffer &buffer); + + QFFmpegAudioInput *audioInput() const { return m_input; } + +private: + QAudioBuffer takeBuffer(); + void retrievePackets(); + + void init() override; + void cleanup() override; + bool hasData() const override; + void processOne() override; + +private: + mutable QMutex m_queueMutex; + std::queue<QAudioBuffer> m_audioBufferQueue; + + AVStream *m_stream = nullptr; + AVCodecContextUPtr m_codecContext; + QFFmpegAudioInput *m_input = nullptr; + QAudioFormat m_format; + + SwrContextUPtr m_resampler; + qint64 m_samplesWritten = 0; + const AVCodec *m_avCodec = nullptr; + QMediaEncoderSettings m_settings; +}; + + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp index 485e39dc9..b6fce596f 100644 --- a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp @@ -7,15 +7,12 @@ #include <qdebug.h> #include <qiodevice.h> -#include <qaudiosource.h> -#include <qaudiobuffer.h> +#include "qffmpegaudioencoder_p.h" #include "qffmpegaudioinput_p.h" #include <private/qplatformcamera_p.h> #include <private/qplatformvideosource_p.h> #include "qffmpegvideobuffer_p.h" #include "qffmpegmediametadata_p.h" -#include "qffmpegencoderoptions_p.h" -#include "qffmpegaudioencoderutils_p.h" #include "qffmpegmuxer_p.h" #include <qloggingcategory.h> @@ -182,204 +179,6 @@ void RecordingEngine::addMediaFrameHandler(Args &&...args) m_connections.append(connection); } -AudioEncoder::AudioEncoder(RecordingEngine *encoder, QFFmpegAudioInput *input, - const QMediaEncoderSettings &settings) - : EncoderThread(encoder), m_input(input), m_settings(settings) -{ - setObjectName(QLatin1String("AudioEncoder")); - qCDebug(qLcFFmpegEncoder) << "AudioEncoder" << settings.audioCodec(); - - m_format = input->device.preferredFormat(); - auto codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(settings.audioCodec()); - Q_ASSERT(avformat_query_codec(encoder->avFormatContext()->oformat, codecID, - FF_COMPLIANCE_NORMAL)); - - const AVAudioFormat requestedAudioFormat(m_format); - - m_avCodec = QFFmpeg::findAVEncoder(codecID, {}, requestedAudioFormat.sampleFormat); - - if (!m_avCodec) - m_avCodec = QFFmpeg::findAVEncoder(codecID); - - qCDebug(qLcFFmpegEncoder) << "found audio codec" << m_avCodec->name; - - Q_ASSERT(m_avCodec); - - m_stream = avformat_new_stream(encoder->avFormatContext(), nullptr); - m_stream->id = encoder->avFormatContext()->nb_streams - 1; - m_stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; - m_stream->codecpar->codec_id = codecID; -#if QT_FFMPEG_OLD_CHANNEL_LAYOUT - m_stream->codecpar->channel_layout = - adjustChannelLayout(m_avCodec->channel_layouts, requestedAudioFormat.channelLayoutMask); - m_stream->codecpar->channels = qPopulationCount(m_stream->codecpar->channel_layout); -#else - m_stream->codecpar->ch_layout = - adjustChannelLayout(m_avCodec->ch_layouts, requestedAudioFormat.channelLayout); -#endif - const auto sampleRate = - adjustSampleRate(m_avCodec->supported_samplerates, requestedAudioFormat.sampleRate); - - m_stream->codecpar->sample_rate = sampleRate; - m_stream->codecpar->frame_size = 1024; - m_stream->codecpar->format = - adjustSampleFormat(m_avCodec->sample_fmts, requestedAudioFormat.sampleFormat); - - m_stream->time_base = AVRational{ 1, sampleRate }; - - qCDebug(qLcFFmpegEncoder) << "set stream time_base" << m_stream->time_base.num << "/" - << m_stream->time_base.den; -} - -void AudioEncoder::open() -{ - m_codecContext.reset(avcodec_alloc_context3(m_avCodec)); - - if (m_stream->time_base.num != 1 || m_stream->time_base.den != m_format.sampleRate()) { - qCDebug(qLcFFmpegEncoder) << "Most likely, av_format_write_header changed time base from" - << 1 << "/" << m_format.sampleRate() << "to" - << m_stream->time_base; - } - - m_codecContext->time_base = m_stream->time_base; - - avcodec_parameters_to_context(m_codecContext.get(), m_stream->codecpar); - - AVDictionaryHolder opts; - applyAudioEncoderOptions(m_settings, m_avCodec->name, m_codecContext.get(), opts); - applyExperimentalCodecOptions(m_avCodec, opts); - - int res = avcodec_open2(m_codecContext.get(), m_avCodec, opts); - qCDebug(qLcFFmpegEncoder) << "audio codec opened" << res; - qCDebug(qLcFFmpegEncoder) << "audio codec params: fmt=" << m_codecContext->sample_fmt - << "rate=" << m_codecContext->sample_rate; - - const AVAudioFormat requestedAudioFormat(m_format); - const AVAudioFormat codecAudioFormat(m_codecContext.get()); - - if (requestedAudioFormat != codecAudioFormat) - m_resampler = createResampleContext(requestedAudioFormat, codecAudioFormat); -} - -void AudioEncoder::addBuffer(const QAudioBuffer &buffer) -{ - QMutexLocker locker(&m_queueMutex); - if (!m_paused.loadRelaxed()) { - m_audioBufferQueue.push(buffer); - locker.unlock(); - dataReady(); - } -} - -QAudioBuffer AudioEncoder::takeBuffer() -{ - QMutexLocker locker(&m_queueMutex); - return dequeueIfPossible(m_audioBufferQueue); -} - -void AudioEncoder::init() -{ - open(); - if (m_input) { - m_input->setFrameSize(m_codecContext->frame_size); - } - qCDebug(qLcFFmpegEncoder) << "AudioEncoder::init started audio device thread."; -} - -void AudioEncoder::cleanup() -{ - while (!m_audioBufferQueue.empty()) - processOne(); - while (avcodec_send_frame(m_codecContext.get(), nullptr) == AVERROR(EAGAIN)) - retrievePackets(); - retrievePackets(); -} - -bool AudioEncoder::hasData() const -{ - QMutexLocker locker(&m_queueMutex); - return !m_audioBufferQueue.empty(); -} - -void AudioEncoder::retrievePackets() -{ - while (1) { - AVPacketUPtr packet(av_packet_alloc()); - int ret = avcodec_receive_packet(m_codecContext.get(), packet.get()); - if (ret < 0) { - if (ret != AVERROR(EOF)) - break; - if (ret != AVERROR(EAGAIN)) { - char errStr[1024]; - av_strerror(ret, errStr, 1024); - qCDebug(qLcFFmpegEncoder) << "receive packet" << ret << errStr; - } - break; - } - - // qCDebug(qLcFFmpegEncoder) << "writing audio packet" << packet->size << packet->pts << packet->dts; - packet->stream_index = m_stream->id; - m_encoder->m_muxer->addPacket(std::move(packet)); - } -} - -void AudioEncoder::processOne() -{ - QAudioBuffer buffer = takeBuffer(); - if (!buffer.isValid()) - return; - - if (buffer.format() != m_format) { - // should we recreate recreate resampler here? - qWarning() << "Get invalid audio format:" << buffer.format() << ", expected:" << m_format; - return; - } - -// qCDebug(qLcFFmpegEncoder) << "new audio buffer" << buffer.byteCount() << buffer.format() << buffer.frameCount() << codec->frame_size; - retrievePackets(); - - auto frame = makeAVFrame(); - frame->format = m_codecContext->sample_fmt; -#if QT_FFMPEG_OLD_CHANNEL_LAYOUT - frame->channel_layout = m_codecContext->channel_layout; - frame->channels = m_codecContext->channels; -#else - frame->ch_layout = m_codecContext->ch_layout; -#endif - frame->sample_rate = m_codecContext->sample_rate; - frame->nb_samples = buffer.frameCount(); - if (frame->nb_samples) - av_frame_get_buffer(frame.get(), 0); - - if (m_resampler) { - const uint8_t *data = buffer.constData<uint8_t>(); - swr_convert(m_resampler.get(), frame->extended_data, frame->nb_samples, &data, - frame->nb_samples); - } else { - memcpy(frame->buf[0]->data, buffer.constData<uint8_t>(), buffer.byteCount()); - } - - const auto &timeBase = m_stream->time_base; - const auto pts = timeBase.den && timeBase.num - ? timeBase.den * m_samplesWritten / (m_codecContext->sample_rate * timeBase.num) - : m_samplesWritten; - setAVFrameTime(*frame, pts, timeBase); - m_samplesWritten += buffer.frameCount(); - - qint64 time = m_format.durationForFrames(m_samplesWritten); - m_encoder->newTimeStamp(time / 1000); - - // qCDebug(qLcFFmpegEncoder) << "sending audio frame" << buffer.byteCount() << frame->pts << - // ((double)buffer.frameCount()/frame->sample_rate); - - int ret = avcodec_send_frame(m_codecContext.get(), frame.get()); - if (ret < 0) { - char errStr[1024]; - av_strerror(ret, errStr, 1024); -// qCDebug(qLcFFmpegEncoder) << "error sending frame" << ret << errStr; - } -} - VideoEncoder::VideoEncoder(RecordingEngine *encoder, const QMediaEncoderSettings &settings, const QVideoFrameFormat &format, std::optional<AVPixelFormat> hwFormat) : EncoderThread(encoder) diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h index 86e322a33..ca8fbf465 100644 --- a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h @@ -131,40 +131,6 @@ protected: RecordingEngine *m_encoder = nullptr; }; -class AudioEncoder : public EncoderThread -{ -public: - AudioEncoder(RecordingEngine *encoder, QFFmpegAudioInput *input, const QMediaEncoderSettings &settings); - - void open(); - void addBuffer(const QAudioBuffer &buffer); - - QFFmpegAudioInput *audioInput() const { return m_input; } - -private: - QAudioBuffer takeBuffer(); - void retrievePackets(); - - void init() override; - void cleanup() override; - bool hasData() const override; - void processOne() override; - -private: - mutable QMutex m_queueMutex; - std::queue<QAudioBuffer> m_audioBufferQueue; - - AVStream *m_stream = nullptr; - AVCodecContextUPtr m_codecContext; - QFFmpegAudioInput *m_input = nullptr; - QAudioFormat m_format; - - SwrContextUPtr m_resampler; - qint64 m_samplesWritten = 0; - const AVCodec *m_avCodec = nullptr; - QMediaEncoderSettings m_settings; -}; - class VideoEncoder : public EncoderThread { public: |