diff options
Diffstat (limited to 'src/plugins/multimedia/ffmpeg/playbackengine')
24 files changed, 3089 insertions, 0 deletions
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp new file mode 100644 index 000000000..64bd82dc0 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp @@ -0,0 +1,407 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegaudiorenderer_p.h" +#include "qaudiosink.h" +#include "qaudiooutput.h" +#include "qaudiobufferoutput.h" +#include "private/qplatformaudiooutput_p.h" +#include <QtCore/qloggingcategory.h> + +#include "qffmpegresampler_p.h" +#include "qffmpegmediaformatinfo_p.h" + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audiorenderer"); + +namespace QFFmpeg { + +using namespace std::chrono_literals; +using namespace std::chrono; + +namespace { +constexpr auto DesiredBufferTime = 110000us; +constexpr auto MinDesiredBufferTime = 22000us; +constexpr auto MaxDesiredBufferTime = 64000us; +constexpr auto MinDesiredFreeBufferTime = 10000us; + +// It might be changed with #ifdef, as on Linux, QPulseAudioSink has quite unstable timings, +// and it needs much more time to make sure that the buffer is overloaded. +constexpr auto BufferLoadingMeasureTime = 400ms; + +constexpr auto DurationBias = 2ms; // avoids extra timer events + +qreal sampleRateFactor() { + // Test purposes: + // + // The env var describes a factor for the sample rate of + // audio data that we feed to the audio sink. + // + // In some cases audio sink might consume data slightly slower or faster than expected; + // even though the synchronization in the audio renderer is supposed to handle it, + // it makes sense to experiment with QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR != 1. + // + // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR > 1 (e.g. 1.01 - 1.1) to test high buffer loading + // or try compensating too fast data consumption by the audio sink. + // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR < 1 to test low buffer loading + // or try compensating too slow data consumption by the audio sink. + + + static const qreal result = []() { + const auto sampleRateFactorStr = qEnvironmentVariable("QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR"); + bool ok = false; + const auto result = sampleRateFactorStr.toDouble(&ok); + return ok ? result : 1.; + }(); + + return result; +} + +QAudioFormat audioFormatFromFrame(const Frame &frame) +{ + return QFFmpegMediaFormatInfo::audioFormatFromCodecParameters( + frame.codec()->stream()->codecpar); +} + +std::unique_ptr<QFFmpegResampler> createResampler(const Frame &frame, + const QAudioFormat &outputFormat) +{ + return std::make_unique<QFFmpegResampler>(frame.codec(), outputFormat, frame.pts()); +} + +} // namespace + +AudioRenderer::AudioRenderer(const TimeController &tc, QAudioOutput *output, + QAudioBufferOutput *bufferOutput) + : Renderer(tc), m_output(output), m_bufferOutput(bufferOutput) +{ + if (output) { + // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294 + connect(output, &QAudioOutput::deviceChanged, this, &AudioRenderer::onDeviceChanged); + connect(output, &QAudioOutput::volumeChanged, this, &AudioRenderer::updateVolume); + connect(output, &QAudioOutput::mutedChanged, this, &AudioRenderer::updateVolume); + } +} + +void AudioRenderer::setOutput(QAudioOutput *output) +{ + setOutputInternal(m_output, output, [this](QAudioOutput *) { onDeviceChanged(); }); +} + +void AudioRenderer::setOutput(QAudioBufferOutput *bufferOutput) +{ + setOutputInternal(m_bufferOutput, bufferOutput, + [this](QAudioBufferOutput *) { m_bufferOutputChanged = true; }); +} + +AudioRenderer::~AudioRenderer() +{ + freeOutput(); +} + +void AudioRenderer::updateVolume() +{ + if (m_sink) + m_sink->setVolume(m_output->isMuted() ? 0.f : m_output->volume()); +} + +void AudioRenderer::onDeviceChanged() +{ + m_deviceChanged = true; +} + +Renderer::RenderingResult AudioRenderer::renderInternal(Frame frame) +{ + if (frame.isValid()) + updateOutputs(frame); + + // push to sink first in order not to waste time on resampling + // for QAudioBufferOutput + const RenderingResult result = pushFrameToOutput(frame); + + if (m_lastFramePushDone) + pushFrameToBufferOutput(frame); + // else // skip pushing the same data to QAudioBufferOutput + + m_lastFramePushDone = result.done; + + return result; +} + +AudioRenderer::RenderingResult AudioRenderer::pushFrameToOutput(const Frame &frame) +{ + if (!m_ioDevice || !m_resampler) + return {}; + + Q_ASSERT(m_sink); + + auto firstFrameFlagGuard = qScopeGuard([&]() { m_firstFrameToSink = false; }); + + const SynchronizationStamp syncStamp{ m_sink->state(), m_sink->bytesFree(), + m_bufferedData.offset, Clock::now() }; + + if (!m_bufferedData.isValid()) { + if (!frame.isValid()) { + if (std::exchange(m_drained, true)) + return {}; + + const auto time = bufferLoadingTime(syncStamp); + + qCDebug(qLcAudioRenderer) << "Draining AudioRenderer, time:" << time; + + return { time.count() == 0, time }; + } + + m_bufferedData = { m_resampler->resample(frame.avFrame()) }; + } + + if (m_bufferedData.isValid()) { + // synchronize after "QIODevice::write" to deliver audio data to the sink ASAP. + auto syncGuard = qScopeGuard([&]() { updateSynchronization(syncStamp, frame); }); + + const auto bytesWritten = m_ioDevice->write(m_bufferedData.data(), m_bufferedData.size()); + + m_bufferedData.offset += bytesWritten; + + if (m_bufferedData.size() <= 0) { + m_bufferedData = {}; + + return {}; + } + + const auto remainingDuration = durationForBytes(m_bufferedData.size()); + + return { false, + std::min(remainingDuration + DurationBias, m_timings.actualBufferDuration / 2) }; + } + + return {}; +} + +void AudioRenderer::pushFrameToBufferOutput(const Frame &frame) +{ + if (!m_bufferOutput) + return; + + Q_ASSERT(m_bufferOutputResampler); + + if (frame.isValid()) { + // TODO: get buffer from m_bufferedData if resample formats are equal + QAudioBuffer buffer = m_resampler->resample(frame.avFrame()); + emit m_bufferOutput->audioBufferReceived(buffer); + } else { + emit m_bufferOutput->audioBufferReceived({}); + } +} + +void AudioRenderer::onPlaybackRateChanged() +{ + m_resampler.reset(); +} + +int AudioRenderer::timerInterval() const +{ + constexpr auto MaxFixableInterval = 50; // ms + + const auto interval = Renderer::timerInterval(); + + if (m_firstFrameToSink || !m_sink || m_sink->state() != QAudio::IdleState + || interval > MaxFixableInterval) + return interval; + + return 0; +} + +void AudioRenderer::onPauseChanged() +{ + m_firstFrameToSink = true; + Renderer::onPauseChanged(); +} + +void AudioRenderer::initResempler(const Frame &frame) +{ + // We recreate resampler whenever format is changed + + auto resamplerFormat = m_sinkFormat; + resamplerFormat.setSampleRate( + qRound(m_sinkFormat.sampleRate() / playbackRate() * sampleRateFactor())); + m_resampler = createResampler(frame, resamplerFormat); +} + +void AudioRenderer::freeOutput() +{ + qCDebug(qLcAudioRenderer) << "Free audio output"; + if (m_sink) { + m_sink->reset(); + + // TODO: inestigate if it's enough to reset the sink without deleting + m_sink.reset(); + } + + m_ioDevice = nullptr; + + m_bufferedData = {}; + m_deviceChanged = false; + m_sinkFormat = {}; + m_timings = {}; + m_bufferLoadingInfo = {}; +} + +void AudioRenderer::updateOutputs(const Frame &frame) +{ + if (m_deviceChanged) { + freeOutput(); + m_resampler.reset(); + } + + if (m_bufferOutput) { + if (m_bufferOutputChanged) { + m_bufferOutputChanged = false; + m_bufferOutputResampler.reset(); + } + + if (!m_bufferOutputResampler) { + QAudioFormat outputFormat = m_bufferOutput->format(); + if (!outputFormat.isValid()) + outputFormat = audioFormatFromFrame(frame); + m_bufferOutputResampler = createResampler(frame, outputFormat); + } + } + + if (!m_output) + return; + + if (!m_sinkFormat.isValid()) { + m_sinkFormat = audioFormatFromFrame(frame); + m_sinkFormat.setChannelConfig(m_output->device().channelConfiguration()); + } + + if (!m_sink) { + // Insert a delay here to test time offset synchronization, e.g. QThread::sleep(1) + m_sink = std::make_unique<QAudioSink>(m_output->device(), m_sinkFormat); + updateVolume(); + m_sink->setBufferSize(m_sinkFormat.bytesForDuration(DesiredBufferTime.count())); + m_ioDevice = m_sink->start(); + m_firstFrameToSink = true; + + connect(m_sink.get(), &QAudioSink::stateChanged, this, + &AudioRenderer::onAudioSinkStateChanged); + + m_timings.actualBufferDuration = durationForBytes(m_sink->bufferSize()); + m_timings.maxSoundDelay = qMin(MaxDesiredBufferTime, + m_timings.actualBufferDuration - MinDesiredFreeBufferTime); + m_timings.minSoundDelay = MinDesiredBufferTime; + + Q_ASSERT(DurationBias < m_timings.minSoundDelay + && m_timings.maxSoundDelay < m_timings.actualBufferDuration); + } + + if (!m_resampler) + initResempler(frame); +} + +void AudioRenderer::updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame) +{ + if (!frame.isValid()) + return; + + Q_ASSERT(m_sink); + + const auto bufferLoadingTime = this->bufferLoadingTime(stamp); + const auto currentFrameDelay = frameDelay(frame, stamp.timePoint); + const auto writtenTime = durationForBytes(stamp.bufferBytesWritten); + const auto soundDelay = currentFrameDelay + bufferLoadingTime - writtenTime; + + auto synchronize = [&](microseconds fixedDelay, microseconds targetSoundDelay) { + // TODO: investigate if we need sample compensation here + + changeRendererTime(fixedDelay - targetSoundDelay); + if (qLcAudioRenderer().isDebugEnabled()) { + // clang-format off + qCDebug(qLcAudioRenderer) + << "Change rendering time:" + << "\n First frame:" << m_firstFrameToSink + << "\n Delay (frame+buffer-written):" << currentFrameDelay << "+" + << bufferLoadingTime << "-" + << writtenTime << "=" + << soundDelay + << "\n Fixed delay:" << fixedDelay + << "\n Target delay:" << targetSoundDelay + << "\n Buffer durations (min/max/limit):" << m_timings.minSoundDelay + << m_timings.maxSoundDelay + << m_timings.actualBufferDuration + << "\n Audio sink state:" << stamp.audioSinkState; + // clang-format on + } + }; + + const auto loadingType = soundDelay > m_timings.maxSoundDelay ? BufferLoadingInfo::High + : soundDelay < m_timings.minSoundDelay ? BufferLoadingInfo::Low + : BufferLoadingInfo::Moderate; + + if (loadingType != m_bufferLoadingInfo.type) { + // qCDebug(qLcAudioRenderer) << "Change buffer loading type:" << + // m_bufferLoadingInfo.type + // << "->" << loadingType << "soundDelay:" << soundDelay; + m_bufferLoadingInfo = { loadingType, stamp.timePoint, soundDelay }; + } + + if (loadingType != BufferLoadingInfo::Moderate) { + const auto isHigh = loadingType == BufferLoadingInfo::High; + const auto shouldHandleIdle = stamp.audioSinkState == QAudio::IdleState && !isHigh; + + auto &fixedDelay = m_bufferLoadingInfo.delay; + + fixedDelay = shouldHandleIdle ? soundDelay + : isHigh ? qMin(soundDelay, fixedDelay) + : qMax(soundDelay, fixedDelay); + + if (stamp.timePoint - m_bufferLoadingInfo.timePoint > BufferLoadingMeasureTime + || (m_firstFrameToSink && isHigh) || shouldHandleIdle) { + const auto targetDelay = isHigh + ? (m_timings.maxSoundDelay + m_timings.minSoundDelay) / 2 + : m_timings.minSoundDelay + DurationBias; + + synchronize(fixedDelay, targetDelay); + m_bufferLoadingInfo = { BufferLoadingInfo::Moderate, stamp.timePoint, targetDelay }; + } + } +} + +microseconds AudioRenderer::bufferLoadingTime(const SynchronizationStamp &syncStamp) const +{ + Q_ASSERT(m_sink); + + if (syncStamp.audioSinkState == QAudio::IdleState) + return microseconds(0); + + const auto bytes = qMax(m_sink->bufferSize() - syncStamp.audioSinkBytesFree, 0); + +#ifdef Q_OS_ANDROID + // The hack has been added due to QAndroidAudioSink issues (QTBUG-118609). + // The method QAndroidAudioSink::bytesFree returns 0 or bufferSize, intermediate values are not + // available now; to be fixed. + if (bytes == 0) + return m_timings.minSoundDelay + MinDesiredBufferTime; +#endif + + return durationForBytes(bytes); +} + +void AudioRenderer::onAudioSinkStateChanged(QAudio::State state) +{ + if (state == QAudio::IdleState && !m_firstFrameToSink) + scheduleNextStep(); +} + +microseconds AudioRenderer::durationForBytes(qsizetype bytes) const +{ + return microseconds(m_sinkFormat.durationForBytes(static_cast<qint32>(bytes))); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegaudiorenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h new file mode 100644 index 000000000..9a22a8a48 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h @@ -0,0 +1,132 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGAUDIORENDERER_P_H +#define QFFMPEGAUDIORENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegrenderer_p.h" + +#include "qaudiobuffer.h" + +QT_BEGIN_NAMESPACE + +class QAudioOutput; +class QAudioBufferOutput; +class QAudioSink; +class QFFmpegResampler; + +namespace QFFmpeg { + +class AudioRenderer : public Renderer +{ + Q_OBJECT +public: + AudioRenderer(const TimeController &tc, QAudioOutput *output, QAudioBufferOutput *bufferOutput); + + void setOutput(QAudioOutput *output); + + void setOutput(QAudioBufferOutput *bufferOutput); + + ~AudioRenderer() override; + +protected: + using Microseconds = std::chrono::microseconds; + struct SynchronizationStamp + { + QAudio::State audioSinkState = QAudio::IdleState; + qsizetype audioSinkBytesFree = 0; + qsizetype bufferBytesWritten = 0; + TimePoint timePoint = TimePoint::max(); + }; + + struct BufferLoadingInfo + { + enum Type { Low, Moderate, High }; + Type type = Moderate; + TimePoint timePoint = TimePoint::max(); + Microseconds delay = Microseconds(0); + }; + + struct AudioTimings + { + Microseconds actualBufferDuration = Microseconds(0); + Microseconds maxSoundDelay = Microseconds(0); + Microseconds minSoundDelay = Microseconds(0); + }; + + struct BufferedDataWithOffset + { + QAudioBuffer buffer; + qsizetype offset = 0; + + bool isValid() const { return buffer.isValid(); } + qsizetype size() const { return buffer.byteCount() - offset; } + const char *data() const { return buffer.constData<char>() + offset; } + }; + + RenderingResult renderInternal(Frame frame) override; + + RenderingResult pushFrameToOutput(const Frame &frame); + + void pushFrameToBufferOutput(const Frame &frame); + + void onPlaybackRateChanged() override; + + int timerInterval() const override; + + void onPauseChanged() override; + + void freeOutput(); + + void updateOutputs(const Frame &frame); + + void initResempler(const Frame &frame); + + void onDeviceChanged(); + + void updateVolume(); + + void updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame); + + Microseconds bufferLoadingTime(const SynchronizationStamp &syncStamp) const; + + void onAudioSinkStateChanged(QAudio::State state); + + Microseconds durationForBytes(qsizetype bytes) const; + +private: + QPointer<QAudioOutput> m_output; + QPointer<QAudioBufferOutput> m_bufferOutput; + std::unique_ptr<QAudioSink> m_sink; + AudioTimings m_timings; + BufferLoadingInfo m_bufferLoadingInfo; + std::unique_ptr<QFFmpegResampler> m_resampler; + std::unique_ptr<QFFmpegResampler> m_bufferOutputResampler; + QAudioFormat m_sinkFormat; + + BufferedDataWithOffset m_bufferedData; + QIODevice *m_ioDevice = nullptr; + + bool m_lastFramePushDone = true; + + bool m_deviceChanged = false; + bool m_bufferOutputChanged = false; + bool m_drained = false; + bool m_firstFrameToSink = true; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGAUDIORENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp new file mode 100644 index 000000000..457b3603d --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp @@ -0,0 +1,82 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegcodec_p.h" +#include "qloggingcategory.h" + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcPlaybackEngineCodec, "qt.multimedia.playbackengine.codec"); + +namespace QFFmpeg { + +Codec::Data::Data(AVCodecContextUPtr context, AVStream *stream, AVFormatContext *formatContext, + std::unique_ptr<QFFmpeg::HWAccel> hwAccel) + : context(std::move(context)), stream(stream), hwAccel(std::move(hwAccel)) +{ + if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) + pixelAspectRatio = av_guess_sample_aspect_ratio(formatContext, stream, nullptr); +} + +QMaybe<Codec> Codec::create(AVStream *stream, AVFormatContext *formatContext) +{ + if (!stream) + return { "Invalid stream" }; + + const AVCodec *decoder = nullptr; + std::unique_ptr<QFFmpeg::HWAccel> hwAccel; + + if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) + std::tie(decoder, hwAccel) = HWAccel::findDecoderWithHwAccel(stream->codecpar->codec_id); + + if (!decoder) + decoder = QFFmpeg::findAVDecoder(stream->codecpar->codec_id); + + if (!decoder) + return { "Failed to find a valid FFmpeg decoder" }; + + qCDebug(qLcPlaybackEngineCodec) << "found decoder" << decoder->name << "for id" << decoder->id; + + AVCodecContextUPtr context(avcodec_alloc_context3(decoder)); + if (!context) + return { "Failed to allocate a FFmpeg codec context" }; + + if (hwAccel) + context->hw_device_ctx = av_buffer_ref(hwAccel->hwDeviceContextAsBuffer()); + + if (context->codec_type != AVMEDIA_TYPE_AUDIO && context->codec_type != AVMEDIA_TYPE_VIDEO + && context->codec_type != AVMEDIA_TYPE_SUBTITLE) { + return { "Unknown codec type" }; + } + + int ret = avcodec_parameters_to_context(context.get(), stream->codecpar); + if (ret < 0) + return { "Failed to set FFmpeg codec parameters" }; + + // ### This still gives errors about wrong HW formats (as we accept all of them) + // But it would be good to get so we can filter out pixel format we don't support natively + context->get_format = QFFmpeg::getFormat; + + /* Init the decoder, with reference counting and threading */ + AVDictionaryHolder opts; + av_dict_set(opts, "refcounted_frames", "1", 0); + av_dict_set(opts, "threads", "auto", 0); + applyExperimentalCodecOptions(decoder, opts); + + ret = avcodec_open2(context.get(), decoder, opts); + if (ret < 0) + return QString("Failed to open FFmpeg codec context " + err2str(ret)); + + return Codec(new Data(std::move(context), stream, formatContext, std::move(hwAccel))); +} + +AVRational Codec::pixelAspectRatio(AVFrame *frame) const +{ + // does the same as av_guess_sample_aspect_ratio, but more efficient + return d->pixelAspectRatio.num && d->pixelAspectRatio.den ? d->pixelAspectRatio + : frame->sample_aspect_ratio; +} + +QT_END_NAMESPACE + +} // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h new file mode 100644 index 000000000..449fb1f65 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h @@ -0,0 +1,62 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGCODEC_P_H +#define QFFMPEGCODEC_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qshareddata.h" +#include "qqueue.h" +#include "private/qmultimediautils_p.h" +#include "qffmpeg_p.h" +#include "qffmpeghwaccel_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class Codec +{ + struct Data + { + Data(AVCodecContextUPtr context, AVStream *stream, AVFormatContext *formatContext, + std::unique_ptr<QFFmpeg::HWAccel> hwAccel); + QAtomicInt ref; + AVCodecContextUPtr context; + AVStream *stream = nullptr; + AVRational pixelAspectRatio = { 0, 1 }; + std::unique_ptr<QFFmpeg::HWAccel> hwAccel; + }; + +public: + static QMaybe<Codec> create(AVStream *stream, AVFormatContext *formatContext); + + AVRational pixelAspectRatio(AVFrame *frame) const; + + AVCodecContext *context() const { return d->context.get(); } + AVStream *stream() const { return d->stream; } + uint streamIndex() const { return d->stream->index; } + HWAccel *hwAccel() const { return d->hwAccel.get(); } + qint64 toMs(qint64 ts) const { return timeStampMs(ts, d->stream->time_base).value_or(0); } + qint64 toUs(qint64 ts) const { return timeStampUs(ts, d->stream->time_base).value_or(0); } + +private: + Codec(Data *data) : d(data) { } + QExplicitlySharedDataPointer<Data> d; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGCODEC_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp new file mode 100644 index 000000000..8cced835c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp @@ -0,0 +1,228 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegdemuxer_p.h" +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +// 4 sec for buffering. TODO: maybe move to env var customization +static constexpr qint64 MaxBufferedDurationUs = 4'000'000; + +// around 4 sec of hdr video +static constexpr qint64 MaxBufferedSize = 32 * 1024 * 1024; + +namespace QFFmpeg { + +static Q_LOGGING_CATEGORY(qLcDemuxer, "qt.multimedia.ffmpeg.demuxer"); + +static qint64 streamTimeToUs(const AVStream *stream, qint64 time) +{ + Q_ASSERT(stream); + + const auto res = mul(time * 1000000, stream->time_base); + return res ? *res : time; +} + +static qint64 packetEndPos(const AVStream *stream, const Packet &packet) +{ + return packet.loopOffset().pos + + streamTimeToUs(stream, packet.avPacket()->pts + packet.avPacket()->duration); +} + +Demuxer::Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops) + : m_context(context), m_posWithOffset(posWithOffset), m_loops(loops) +{ + qCDebug(qLcDemuxer) << "Create demuxer." + << "pos:" << posWithOffset.pos << "loop offset:" << posWithOffset.offset.pos + << "loop index:" << posWithOffset.offset.index << "loops:" << loops; + + Q_ASSERT(m_context); + + for (auto i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) { + if (streamIndexes[i] >= 0) { + const auto trackType = static_cast<QPlatformMediaPlayer::TrackType>(i); + qCDebug(qLcDemuxer) << "Activate demuxing stream" << i << ", trackType:" << trackType; + m_streams[streamIndexes[i]] = { trackType }; + } + } +} + +void Demuxer::doNextStep() +{ + ensureSeeked(); + + Packet packet(m_posWithOffset.offset, AVPacketUPtr{ av_packet_alloc() }, id()); + if (av_read_frame(m_context, packet.avPacket()) < 0) { + ++m_posWithOffset.offset.index; + + const auto loops = m_loops.loadAcquire(); + if (loops >= 0 && m_posWithOffset.offset.index >= loops) { + qCDebug(qLcDemuxer) << "finish demuxing"; + + if (!std::exchange(m_buffered, true)) + emit packetsBuffered(); + + setAtEnd(true); + } else { + m_seeked = false; + m_posWithOffset.pos = 0; + m_posWithOffset.offset.pos = m_maxPacketsEndPos; + m_maxPacketsEndPos = 0; + + ensureSeeked(); + + qCDebug(qLcDemuxer) << "Demuxer loops changed. Index:" << m_posWithOffset.offset.index + << "Offset:" << m_posWithOffset.offset.pos; + + scheduleNextStep(false); + } + + return; + } + + auto &avPacket = *packet.avPacket(); + + const auto streamIndex = avPacket.stream_index; + const auto stream = m_context->streams[streamIndex]; + + auto it = m_streams.find(streamIndex); + if (it != m_streams.end()) { + auto &streamData = it->second; + + const auto endPos = packetEndPos(stream, packet); + m_maxPacketsEndPos = qMax(m_maxPacketsEndPos, endPos); + + // Increase buffered metrics as the packet has been processed. + + streamData.bufferedDuration += streamTimeToUs(stream, avPacket.duration); + streamData.bufferedSize += avPacket.size; + streamData.maxSentPacketsPos = qMax(streamData.maxSentPacketsPos, endPos); + updateStreamDataLimitFlag(streamData); + + if (!m_buffered && streamData.isDataLimitReached) { + m_buffered = true; + emit packetsBuffered(); + } + + if (!m_firstPacketFound) { + m_firstPacketFound = true; + const auto pos = streamTimeToUs(stream, avPacket.pts); + emit firstPacketFound(std::chrono::steady_clock::now(), pos); + } + + auto signal = signalByTrackType(it->second.trackType); + emit (this->*signal)(packet); + } + + scheduleNextStep(false); +} + +void Demuxer::onPacketProcessed(Packet packet) +{ + Q_ASSERT(packet.isValid()); + + if (packet.sourceId() != id()) + return; + + auto &avPacket = *packet.avPacket(); + + const auto streamIndex = avPacket.stream_index; + const auto stream = m_context->streams[streamIndex]; + auto it = m_streams.find(streamIndex); + + if (it != m_streams.end()) { + auto &streamData = it->second; + + // Decrease buffered metrics as new data (the packet) has been received (buffered) + + streamData.bufferedDuration -= streamTimeToUs(stream, avPacket.duration); + streamData.bufferedSize -= avPacket.size; + streamData.maxProcessedPacketPos = + qMax(streamData.maxProcessedPacketPos, packetEndPos(stream, packet)); + + Q_ASSERT(it->second.bufferedDuration >= 0); + Q_ASSERT(it->second.bufferedSize >= 0); + + updateStreamDataLimitFlag(streamData); + } + + scheduleNextStep(); +} + +bool Demuxer::canDoNextStep() const +{ + auto isDataLimitReached = [](const auto &streamIndexToData) { + return streamIndexToData.second.isDataLimitReached; + }; + + // Demuxer waits: + // - if it's paused + // - if the end has been reached + // - if streams are empty (probably, should be handled on the initialization) + // - if at least one of the streams has reached the data limit (duration or size) + + return PlaybackEngineObject::canDoNextStep() && !isAtEnd() && !m_streams.empty() + && std::none_of(m_streams.begin(), m_streams.end(), isDataLimitReached); +} + +void Demuxer::ensureSeeked() +{ + if (std::exchange(m_seeked, true)) + return; + + if ((m_context->ctx_flags & AVFMTCTX_UNSEEKABLE) == 0) { + const qint64 seekPos = m_posWithOffset.pos * AV_TIME_BASE / 1000000; + auto err = av_seek_frame(m_context, -1, seekPos, AVSEEK_FLAG_BACKWARD); + + if (err < 0) { + qCWarning(qLcDemuxer) << "Failed to seek, pos" << seekPos; + + // Drop an error of seeking to initial position of streams with undefined duration. + // This needs improvements. + if (seekPos != 0 || m_context->duration > 0) + emit error(QMediaPlayer::ResourceError, + QLatin1StringView("Failed to seek: ") + err2str(err)); + } + } + + setAtEnd(false); +} + +Demuxer::RequestingSignal Demuxer::signalByTrackType(QPlatformMediaPlayer::TrackType trackType) +{ + switch (trackType) { + case QPlatformMediaPlayer::TrackType::VideoStream: + return &Demuxer::requestProcessVideoPacket; + case QPlatformMediaPlayer::TrackType::AudioStream: + return &Demuxer::requestProcessAudioPacket; + case QPlatformMediaPlayer::TrackType::SubtitleStream: + return &Demuxer::requestProcessSubtitlePacket; + default: + Q_ASSERT(!"Unknown track type"); + } + + return nullptr; +} + +void Demuxer::setLoops(int loopsCount) +{ + qCDebug(qLcDemuxer) << "setLoops to demuxer" << loopsCount; + m_loops.storeRelease(loopsCount); +} + +void Demuxer::updateStreamDataLimitFlag(StreamData &streamData) +{ + const auto packetsPosDiff = streamData.maxSentPacketsPos - streamData.maxProcessedPacketPos; + streamData.isDataLimitReached = + streamData.bufferedDuration >= MaxBufferedDurationUs + || (streamData.bufferedDuration == 0 && packetsPosDiff >= MaxBufferedDurationUs) + || streamData.bufferedSize >= MaxBufferedSize; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegdemuxer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h new file mode 100644 index 000000000..b72056185 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h @@ -0,0 +1,87 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGDEMUXER_P_H +#define QFFMPEGDEMUXER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegplaybackengineobject_p.h" +#include "private/qplatformmediaplayer_p.h" +#include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" + +#include <unordered_map> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class Demuxer : public PlaybackEngineObject +{ + Q_OBJECT +public: + Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops); + + using RequestingSignal = void (Demuxer::*)(Packet); + static RequestingSignal signalByTrackType(QPlatformMediaPlayer::TrackType trackType); + + void setLoops(int loopsCount); + +public slots: + void onPacketProcessed(Packet); + +signals: + void requestProcessAudioPacket(Packet); + void requestProcessVideoPacket(Packet); + void requestProcessSubtitlePacket(Packet); + void firstPacketFound(TimePoint tp, qint64 trackPos); + void packetsBuffered(); + +private: + bool canDoNextStep() const override; + + void doNextStep() override; + + void ensureSeeked(); + +private: + struct StreamData + { + QPlatformMediaPlayer::TrackType trackType = QPlatformMediaPlayer::TrackType::NTrackTypes; + qint64 bufferedDuration = 0; + qint64 bufferedSize = 0; + + qint64 maxSentPacketsPos = 0; + qint64 maxProcessedPacketPos = 0; + + bool isDataLimitReached = false; + }; + + void updateStreamDataLimitFlag(StreamData &streamData); + +private: + AVFormatContext *m_context = nullptr; + bool m_seeked = false; + bool m_firstPacketFound = false; + std::unordered_map<int, StreamData> m_streams; + PositionWithOffset m_posWithOffset; + qint64 m_maxPacketsEndPos = 0; + QAtomicInt m_loops = QMediaPlayer::Once; + bool m_buffered = false; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE // QFFMPEGDEMUXER_P_H + +#endif diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h new file mode 100644 index 000000000..84fe2fead --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h @@ -0,0 +1,109 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGFRAME_P_H +#define QFFMPEGFRAME_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeg_p.h" +#include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" +#include "QtCore/qsharedpointer.h" +#include "qpointer.h" +#include "qobject.h" + +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct Frame +{ + struct Data + { + Data(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64, quint64 sourceId) + : loopOffset(offset), codec(codec), frame(std::move(f)), sourceId(sourceId) + { + Q_ASSERT(frame); + if (frame->pts != AV_NOPTS_VALUE) + pts = codec.toUs(frame->pts); + else + pts = codec.toUs(frame->best_effort_timestamp); + + if (auto frameDuration = getAVFrameDuration(*frame)) { + duration = codec.toUs(frameDuration); + } else { + const auto &avgFrameRate = codec.stream()->avg_frame_rate; + duration = mul(qint64(1000000), { avgFrameRate.den, avgFrameRate.num }).value_or(0); + } + } + Data(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + quint64 sourceId) + : loopOffset(offset), text(text), pts(pts), duration(duration), sourceId(sourceId) + { + } + + QAtomicInt ref; + LoopOffset loopOffset; + std::optional<Codec> codec; + AVFrameUPtr frame; + QString text; + qint64 pts = -1; + qint64 duration = -1; + quint64 sourceId = 0; + }; + Frame() = default; + + Frame(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64 pts, + quint64 sourceIndex) + : d(new Data(offset, std::move(f), codec, pts, sourceIndex)) + { + } + Frame(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + quint64 sourceIndex) + : d(new Data(offset, text, pts, duration, sourceIndex)) + { + } + bool isValid() const { return !!d; } + + AVFrame *avFrame() const { return data().frame.get(); } + AVFrameUPtr takeAVFrame() { return std::move(data().frame); } + const Codec *codec() const { return data().codec ? &data().codec.value() : nullptr; } + qint64 pts() const { return data().pts; } + qint64 duration() const { return data().duration; } + qint64 end() const { return data().pts + data().duration; } + QString text() const { return data().text; } + quint64 sourceId() const { return data().sourceId; }; + const LoopOffset &loopOffset() const { return data().loopOffset; }; + qint64 absolutePts() const { return pts() + loopOffset().pos; } + qint64 absoluteEnd() const { return end() + loopOffset().pos; } + +private: + Data &data() const + { + Q_ASSERT(d); + return *d; + } + +private: + QExplicitlySharedDataPointer<Data> d; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +Q_DECLARE_METATYPE(QFFmpeg::Frame); + +#endif // QFFMPEGFRAME_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder.cpp new file mode 100644 index 000000000..f92f93ddb --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder.cpp @@ -0,0 +1,390 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegmediadataholder_p.h" + +#include "qffmpegmediametadata_p.h" +#include "qffmpegmediaformatinfo_p.h" +#include "qffmpegioutils_p.h" +#include "qiodevice.h" +#include "qdatetime.h" +#include "qloggingcategory.h" + +#include <math.h> +#include <optional> + +extern "C" { +#include "libavutil/display.h" +} + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcMediaDataHolder, "qt.multimedia.ffmpeg.mediadataholder") + +namespace QFFmpeg { + +static std::optional<qint64> streamDuration(const AVStream &stream) +{ + const auto &factor = stream.time_base; + + if (stream.duration > 0 && factor.num > 0 && factor.den > 0) { + return qint64(1000000) * stream.duration * factor.num / factor.den; + } + + // In some cases ffmpeg reports negative duration that is definitely invalid. + // However, the correct duration may be read from the metadata. + + if (stream.duration < 0) { + qCWarning(qLcMediaDataHolder) << "AVStream duration" << stream.duration + << "is invalid. Taking it from the metadata"; + } + + if (const auto duration = av_dict_get(stream.metadata, "DURATION", nullptr, 0)) { + const auto time = QTime::fromString(QString::fromUtf8(duration->value)); + return qint64(1000) * time.msecsSinceStartOfDay(); + } + + return {}; +} + +static int streamOrientation(const AVStream *stream) +{ + Q_ASSERT(stream); + + using SideDataSize = decltype(AVPacketSideData::size); + constexpr SideDataSize displayMatrixSize = sizeof(int32_t) * 9; + const auto *sideData = streamSideData(stream, AV_PKT_DATA_DISPLAYMATRIX); + if (!sideData || sideData->size < displayMatrixSize) + return 0; + + auto displayMatrix = reinterpret_cast<const int32_t *>(sideData->data); + auto rotation = static_cast<int>(std::round(av_display_rotation_get(displayMatrix))); + // Convert counterclockwise rotation angle to clockwise, restricted to 0, 90, 180 and 270 + if (rotation % 90 != 0) + return 0; + return rotation < 0 ? -rotation % 360 : -rotation % 360 + 360; +} + + +static bool colorTransferSupportsHdr(const AVStream *stream) +{ + if (!stream) + return false; + + const AVCodecParameters *codecPar = stream->codecpar; + if (!codecPar) + return false; + + const QVideoFrameFormat::ColorTransfer colorTransfer = fromAvColorTransfer(codecPar->color_trc); + + // Assume that content is using HDR if the color transfer supports high + // dynamic range. The video may still not utilize the extended range, + // but we can't determine the actual range without decoding frames. + return colorTransfer == QVideoFrameFormat::ColorTransfer_ST2084 + || colorTransfer == QVideoFrameFormat::ColorTransfer_STD_B67; +} + +QtVideo::Rotation MediaDataHolder::rotation() const +{ + int orientation = m_metaData.value(QMediaMetaData::Orientation).toInt(); + return static_cast<QtVideo::Rotation>(orientation); +} + +AVFormatContext *MediaDataHolder::avContext() +{ + return m_context.get(); +} + +int MediaDataHolder::currentStreamIndex(QPlatformMediaPlayer::TrackType trackType) const +{ + return m_currentAVStreamIndex[trackType]; +} + +static void insertMediaData(QMediaMetaData &metaData, QPlatformMediaPlayer::TrackType trackType, + const AVStream *stream) +{ + Q_ASSERT(stream); + const auto *codecPar = stream->codecpar; + + switch (trackType) { + case QPlatformMediaPlayer::VideoStream: + metaData.insert(QMediaMetaData::VideoBitRate, (int)codecPar->bit_rate); + metaData.insert(QMediaMetaData::VideoCodec, + QVariant::fromValue(QFFmpegMediaFormatInfo::videoCodecForAVCodecId( + codecPar->codec_id))); + metaData.insert(QMediaMetaData::Resolution, QSize(codecPar->width, codecPar->height)); + metaData.insert(QMediaMetaData::VideoFrameRate, + qreal(stream->avg_frame_rate.num) / qreal(stream->avg_frame_rate.den)); + metaData.insert(QMediaMetaData::Orientation, QVariant::fromValue(streamOrientation(stream))); + metaData.insert(QMediaMetaData::HasHdrContent, colorTransferSupportsHdr(stream)); + break; + case QPlatformMediaPlayer::AudioStream: + metaData.insert(QMediaMetaData::AudioBitRate, (int)codecPar->bit_rate); + metaData.insert(QMediaMetaData::AudioCodec, + QVariant::fromValue(QFFmpegMediaFormatInfo::audioCodecForAVCodecId( + codecPar->codec_id))); + break; + default: + break; + } +}; + +QPlatformMediaPlayer::TrackType MediaDataHolder::trackTypeFromMediaType(int mediaType) +{ + switch (mediaType) { + case AVMEDIA_TYPE_AUDIO: + return QPlatformMediaPlayer::AudioStream; + case AVMEDIA_TYPE_VIDEO: + return QPlatformMediaPlayer::VideoStream; + case AVMEDIA_TYPE_SUBTITLE: + return QPlatformMediaPlayer::SubtitleStream; + default: + return QPlatformMediaPlayer::NTrackTypes; + } +} + +namespace { +QMaybe<AVFormatContextUPtr, MediaDataHolder::ContextError> +loadMedia(const QUrl &mediaUrl, QIODevice *stream, const std::shared_ptr<ICancelToken> &cancelToken) +{ + const QByteArray url = mediaUrl.toString(QUrl::PreferLocalFile).toUtf8(); + + AVFormatContextUPtr context{ avformat_alloc_context() }; + + if (stream) { + if (!stream->isOpen()) { + if (!stream->open(QIODevice::ReadOnly)) + return MediaDataHolder::ContextError{ + QMediaPlayer::ResourceError, QLatin1String("Could not open source device.") + }; + } + if (!stream->isSequential()) + stream->seek(0); + + constexpr int bufferSize = 32768; + unsigned char *buffer = (unsigned char *)av_malloc(bufferSize); + context->pb = avio_alloc_context(buffer, bufferSize, false, stream, &readQIODevice, nullptr, + &seekQIODevice); + } + + AVDictionaryHolder dict; + constexpr auto NetworkTimeoutUs = "5000000"; + av_dict_set(dict, "timeout", NetworkTimeoutUs, 0); + + const QByteArray protocolWhitelist = qgetenv("QT_FFMPEG_PROTOCOL_WHITELIST"); + if (!protocolWhitelist.isNull()) + av_dict_set(dict, "protocol_whitelist", protocolWhitelist.data(), 0); + + context->interrupt_callback.opaque = cancelToken.get(); + context->interrupt_callback.callback = [](void *opaque) { + const auto *cancelToken = static_cast<const ICancelToken *>(opaque); + if (cancelToken && cancelToken->isCancelled()) + return 1; + return 0; + }; + + int ret = 0; + { + AVFormatContext *contextRaw = context.release(); + ret = avformat_open_input(&contextRaw, url.constData(), nullptr, dict); + context.reset(contextRaw); + } + + if (ret < 0) { + auto code = QMediaPlayer::ResourceError; + if (ret == AVERROR(EACCES)) + code = QMediaPlayer::AccessDeniedError; + else if (ret == AVERROR(EINVAL)) + code = QMediaPlayer::FormatError; + + return MediaDataHolder::ContextError{ code, QMediaPlayer::tr("Could not open file") }; + } + + ret = avformat_find_stream_info(context.get(), nullptr); + if (ret < 0) { + return MediaDataHolder::ContextError{ + QMediaPlayer::FormatError, + QMediaPlayer::tr("Could not find stream information for media file") + }; + } + +#ifndef QT_NO_DEBUG + av_dump_format(context.get(), 0, url.constData(), 0); +#endif + return context; +} + +} // namespace + +MediaDataHolder::Maybe MediaDataHolder::create(const QUrl &url, QIODevice *stream, + const std::shared_ptr<ICancelToken> &cancelToken) +{ + QMaybe context = loadMedia(url, stream, cancelToken); + if (context) { + // MediaDataHolder is wrapped in a shared pointer to interop with signal/slot mechanism + return QSharedPointer<MediaDataHolder>{ new MediaDataHolder{ std::move(context.value()), cancelToken } }; + } + return context.error(); +} + +MediaDataHolder::MediaDataHolder(AVFormatContextUPtr context, + const std::shared_ptr<ICancelToken> &cancelToken) + : m_cancelToken{ cancelToken } +{ + Q_ASSERT(context); + + m_context = std::move(context); + m_isSeekable = !(m_context->ctx_flags & AVFMTCTX_UNSEEKABLE); + + for (unsigned int i = 0; i < m_context->nb_streams; ++i) { + + const auto *stream = m_context->streams[i]; + const auto trackType = trackTypeFromMediaType(stream->codecpar->codec_type); + + if (trackType == QPlatformMediaPlayer::NTrackTypes) + continue; + + if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC) + continue; // Ignore attached picture streams because we treat them as metadata + + auto metaData = QFFmpegMetaData::fromAVMetaData(stream->metadata); + const bool isDefault = stream->disposition & AV_DISPOSITION_DEFAULT; + + if (trackType != QPlatformMediaPlayer::SubtitleStream) { + insertMediaData(metaData, trackType, stream); + + if (isDefault && m_requestedStreams[trackType] < 0) + m_requestedStreams[trackType] = m_streamMap[trackType].size(); + } + + if (auto duration = streamDuration(*stream)) { + m_duration = qMax(m_duration, *duration); + metaData.insert(QMediaMetaData::Duration, *duration / qint64(1000)); + } + + m_streamMap[trackType].append({ (int)i, isDefault, metaData }); + } + + // With some media files, streams may be lacking duration info. Let's + // get it from ffmpeg's duration estimation instead. + if (m_duration == 0 && m_context->duration > 0ll) { + m_duration = m_context->duration; + } + + for (auto trackType : + { QPlatformMediaPlayer::VideoStream, QPlatformMediaPlayer::AudioStream }) { + auto &requestedStream = m_requestedStreams[trackType]; + auto &streamMap = m_streamMap[trackType]; + + if (requestedStream < 0 && !streamMap.empty()) + requestedStream = 0; + + if (requestedStream >= 0) + m_currentAVStreamIndex[trackType] = streamMap[requestedStream].avStreamIndex; + } + + updateMetaData(); +} + +namespace { + +/*! + \internal + + Attempt to find an attached picture from the context's streams. + This will find ID3v2 pictures on audio files, and also pictures + attached to videos. + */ +QImage getAttachedPicture(const AVFormatContext *context) +{ + if (!context) + return {}; + + for (unsigned int i = 0; i < context->nb_streams; ++i) { + const AVStream* stream = context->streams[i]; + if (!stream || !(stream->disposition & AV_DISPOSITION_ATTACHED_PIC)) + continue; + + const AVPacket *compressedImage = &stream->attached_pic; + if (!compressedImage || !compressedImage->data || compressedImage->size <= 0) + continue; + + // Feed raw compressed data to QImage::fromData, which will decompress it + // if it is a recognized format. + QImage image = QImage::fromData({ compressedImage->data, compressedImage->size }); + if (!image.isNull()) + return image; + } + + return {}; +} + +} + +void MediaDataHolder::updateMetaData() +{ + m_metaData = {}; + + if (!m_context) + return; + + m_metaData = QFFmpegMetaData::fromAVMetaData(m_context->metadata); + m_metaData.insert(QMediaMetaData::FileFormat, + QVariant::fromValue(QFFmpegMediaFormatInfo::fileFormatForAVInputFormat( + m_context->iformat))); + m_metaData.insert(QMediaMetaData::Duration, m_duration / qint64(1000)); + + if (!m_cachedThumbnail.has_value()) + m_cachedThumbnail = getAttachedPicture(m_context.get()); + + if (!m_cachedThumbnail->isNull()) + m_metaData.insert(QMediaMetaData::ThumbnailImage, m_cachedThumbnail.value()); + + for (auto trackType : + { QPlatformMediaPlayer::AudioStream, QPlatformMediaPlayer::VideoStream }) { + const auto streamIndex = m_currentAVStreamIndex[trackType]; + if (streamIndex >= 0) + insertMediaData(m_metaData, trackType, m_context->streams[streamIndex]); + } +} + +bool MediaDataHolder::setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber) +{ + if (!m_context) + return false; + + if (streamNumber < 0 || streamNumber >= m_streamMap[type].size()) + streamNumber = -1; + if (m_requestedStreams[type] == streamNumber) + return false; + m_requestedStreams[type] = streamNumber; + const int avStreamIndex = m_streamMap[type].value(streamNumber).avStreamIndex; + + const int oldIndex = m_currentAVStreamIndex[type]; + qCDebug(qLcMediaDataHolder) << ">>>>> change track" << type << "from" << oldIndex << "to" + << avStreamIndex; + + // TODO: maybe add additional verifications + m_currentAVStreamIndex[type] = avStreamIndex; + + updateMetaData(); + + return true; +} + +int MediaDataHolder::activeTrack(QPlatformMediaPlayer::TrackType type) const +{ + return type < QPlatformMediaPlayer::NTrackTypes ? m_requestedStreams[type] : -1; +} + +const QList<MediaDataHolder::StreamInfo> &MediaDataHolder::streamInfo( + QPlatformMediaPlayer::TrackType trackType) const +{ + Q_ASSERT(trackType < QPlatformMediaPlayer::NTrackTypes); + + return m_streamMap[trackType]; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder_p.h new file mode 100644 index 000000000..a55b0766a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder_p.h @@ -0,0 +1,107 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGMEDIADATAHOLDER_P_H +#define QFFMPEGMEDIADATAHOLDER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qmediametadata.h" +#include "private/qplatformmediaplayer_p.h" +#include "qffmpeg_p.h" +#include "qvideoframe.h" +#include <private/qmultimediautils_p.h> + +#include <array> +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct ICancelToken +{ + virtual ~ICancelToken() = default; + virtual bool isCancelled() const = 0; +}; + +using AVFormatContextUPtr = std::unique_ptr<AVFormatContext, AVDeleter<decltype(&avformat_close_input), &avformat_close_input>>; + +class MediaDataHolder +{ +public: + struct StreamInfo + { + int avStreamIndex = -1; + bool isDefault = false; + QMediaMetaData metaData; + }; + + struct ContextError + { + int code = 0; + QString description; + }; + + using StreamsMap = std::array<QList<StreamInfo>, QPlatformMediaPlayer::NTrackTypes>; + using StreamIndexes = std::array<int, QPlatformMediaPlayer::NTrackTypes>; + + MediaDataHolder() = default; + MediaDataHolder(AVFormatContextUPtr context, const std::shared_ptr<ICancelToken> &cancelToken); + + static QPlatformMediaPlayer::TrackType trackTypeFromMediaType(int mediaType); + + int activeTrack(QPlatformMediaPlayer::TrackType type) const; + + const QList<StreamInfo> &streamInfo(QPlatformMediaPlayer::TrackType trackType) const; + + qint64 duration() const { return m_duration; } + + const QMediaMetaData &metaData() const { return m_metaData; } + + bool isSeekable() const { return m_isSeekable; } + + QtVideo::Rotation rotation() const; + + AVFormatContext *avContext(); + + int currentStreamIndex(QPlatformMediaPlayer::TrackType trackType) const; + + using Maybe = QMaybe<QSharedPointer<MediaDataHolder>, ContextError>; + static Maybe create(const QUrl &url, QIODevice *stream, + const std::shared_ptr<ICancelToken> &cancelToken); + + bool setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber); + +private: + void updateMetaData(); + + std::shared_ptr<ICancelToken> m_cancelToken; // NOTE: Cancel token may be accessed by + // AVFormatContext during destruction and + // must outlive the context object + AVFormatContextUPtr m_context; + + bool m_isSeekable = false; + + StreamIndexes m_currentAVStreamIndex = { -1, -1, -1 }; + StreamsMap m_streamMap; + StreamIndexes m_requestedStreams = { -1, -1, -1 }; + qint64 m_duration = 0; + QMediaMetaData m_metaData; + std::optional<QImage> m_cachedThumbnail; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGMEDIADATAHOLDER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h new file mode 100644 index 000000000..5e15bf012 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h @@ -0,0 +1,61 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGPACKET_P_H +#define QFFMPEGPACKET_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeg_p.h" +#include "QtCore/qsharedpointer.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct Packet +{ + struct Data + { + Data(const LoopOffset &offset, AVPacketUPtr p, quint64 sourceId) + : loopOffset(offset), packet(std::move(p)), sourceId(sourceId) + { + } + + QAtomicInt ref; + LoopOffset loopOffset; + AVPacketUPtr packet; + quint64 sourceId; + }; + Packet() = default; + Packet(const LoopOffset &offset, AVPacketUPtr p, quint64 sourceId) + : d(new Data(offset, std::move(p), sourceId)) + { + } + + bool isValid() const { return !!d; } + AVPacket *avPacket() const { return d->packet.get(); } + const LoopOffset &loopOffset() const { return d->loopOffset; } + quint64 sourceId() const { return d->sourceId; } + +private: + QExplicitlySharedDataPointer<Data> d; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +Q_DECLARE_METATYPE(QFFmpeg::Packet) + +#endif // QFFMPEGPACKET_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackenginedefs_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackenginedefs_p.h new file mode 100644 index 000000000..18254ef64 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackenginedefs_p.h @@ -0,0 +1,46 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGPLAYBACKENGINEDEFS_P_H +#define QFFMPEGPLAYBACKENGINEDEFS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// +#include "qobject.h" +#include "qpointer.h" + +#include <memory> +#include <array> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { +class PlaybackEngine; +} + +namespace QFFmpeg { + +using StreamIndexes = std::array<int, 3>; + +class PlaybackEngineObjectsController; +class PlaybackEngineObject; +class Demuxer; +class StreamDecoder; +class Renderer; +class SubtitleRenderer; +class AudioRenderer; +class VideoRenderer; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGPLAYBACKENGINEDEFS_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject.cpp new file mode 100644 index 000000000..2d23802de --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject.cpp @@ -0,0 +1,109 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegplaybackengineobject_p.h" + +#include "qtimer.h" +#include "qdebug.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +static QAtomicInteger<PlaybackEngineObject::Id> PersistentId = 0; + +PlaybackEngineObject::PlaybackEngineObject() : m_id(PersistentId.fetchAndAddRelaxed(1)) { } + +PlaybackEngineObject::~PlaybackEngineObject() +{ + if (thread() != QThread::currentThread()) + qWarning() << "The playback engine object is being removed in an unexpected thread"; +} + +bool PlaybackEngineObject::isPaused() const +{ + return m_paused; +} + +void PlaybackEngineObject::setAtEnd(bool isAtEnd) +{ + if (m_atEnd.testAndSetRelease(!isAtEnd, isAtEnd) && isAtEnd) + emit atEnd(); +} + +bool PlaybackEngineObject::isAtEnd() const +{ + return m_atEnd; +} + +PlaybackEngineObject::Id PlaybackEngineObject::id() const +{ + return m_id; +} + +void PlaybackEngineObject::setPaused(bool isPaused) +{ + if (m_paused.testAndSetRelease(!isPaused, isPaused)) + QMetaObject::invokeMethod(this, &PlaybackEngineObject::onPauseChanged); +} + +void PlaybackEngineObject::kill() +{ + m_deleting.storeRelease(true); + + disconnect(); + deleteLater(); +} + +bool PlaybackEngineObject::canDoNextStep() const +{ + return !m_paused; +} + +QTimer &PlaybackEngineObject::timer() +{ + if (!m_timer) { + m_timer = std::make_unique<QTimer>(); + m_timer->setTimerType(Qt::PreciseTimer); + m_timer->setSingleShot(true); + connect(m_timer.get(), &QTimer::timeout, this, &PlaybackEngineObject::onTimeout); + } + + return *m_timer; +} + +void PlaybackEngineObject::onTimeout() +{ + if (!m_deleting && canDoNextStep()) + doNextStep(); +} + +int PlaybackEngineObject::timerInterval() const +{ + return 0; +} + +void PlaybackEngineObject::onPauseChanged() +{ + scheduleNextStep(); +} + +void PlaybackEngineObject::scheduleNextStep(bool allowDoImmediatelly) +{ + if (!m_deleting && canDoNextStep()) { + const auto interval = timerInterval(); + if (interval == 0 && allowDoImmediatelly) { + timer().stop(); + doNextStep(); + } else { + timer().start(interval); + } + } else { + timer().stop(); + } +} +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegplaybackengineobject_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject_p.h new file mode 100644 index 000000000..02943a55b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject_p.h @@ -0,0 +1,84 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGPLAYBACKENGINEOBJECT_P_H +#define QFFMPEGPLAYBACKENGINEOBJECT_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegplaybackenginedefs_p.h" +#include "qthread.h" +#include "qatomic.h" + +QT_BEGIN_NAMESPACE + +class QTimer; + +namespace QFFmpeg { + +class PlaybackEngineObject : public QObject +{ + Q_OBJECT +public: + using TimePoint = std::chrono::steady_clock::time_point; + using TimePointOpt = std::optional<TimePoint>; + using Id = quint64; + + PlaybackEngineObject(); + + ~PlaybackEngineObject(); + + bool isPaused() const; + + bool isAtEnd() const; + + void kill(); + + void setPaused(bool isPaused); + + Id id() const; + +signals: + void atEnd(); + + void error(int code, const QString &errorString); + +protected: + QTimer &timer(); + + void scheduleNextStep(bool allowDoImmediatelly = true); + + virtual void onPauseChanged(); + + virtual bool canDoNextStep() const; + + virtual int timerInterval() const; + + void setAtEnd(bool isAtEnd); + + virtual void doNextStep() { } + +private slots: + void onTimeout(); + +private: + std::unique_ptr<QTimer> m_timer; + + QAtomicInteger<bool> m_paused = true; + QAtomicInteger<bool> m_atEnd = false; + QAtomicInteger<bool> m_deleting = false; + const Id m_id; +}; +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGPLAYBACKENGINEOBJECT_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h new file mode 100644 index 000000000..a30fdc119 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h @@ -0,0 +1,40 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef qffmpegpositionwithoffset_p_H +#define qffmpegpositionwithoffset_p_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtypes.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct LoopOffset +{ + qint64 pos = 0; + int index = 0; +}; + +struct PositionWithOffset +{ + qint64 pos = 0; + LoopOffset offset; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // qffmpegpositionwithoffset_p_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp new file mode 100644 index 000000000..e763c786b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp @@ -0,0 +1,216 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegrenderer_p.h" +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +static Q_LOGGING_CATEGORY(qLcRenderer, "qt.multimedia.ffmpeg.renderer"); + +Renderer::Renderer(const TimeController &tc, const std::chrono::microseconds &seekPosTimeOffset) + : m_timeController(tc), + m_lastFrameEnd(tc.currentPosition()), + m_lastPosition(m_lastFrameEnd), + m_seekPos(tc.currentPosition(-seekPosTimeOffset)) +{ +} + +void Renderer::syncSoft(TimePoint tp, qint64 trackTime) +{ + QMetaObject::invokeMethod(this, [this, tp, trackTime]() { + m_timeController.syncSoft(tp, trackTime); + scheduleNextStep(true); + }); +} + +qint64 Renderer::seekPosition() const +{ + return m_seekPos; +} + +qint64 Renderer::lastPosition() const +{ + return m_lastPosition; +} + +void Renderer::setPlaybackRate(float rate) +{ + QMetaObject::invokeMethod(this, [this, rate]() { + m_timeController.setPlaybackRate(rate); + onPlaybackRateChanged(); + scheduleNextStep(); + }); +} + +void Renderer::doForceStep() +{ + if (m_isStepForced.testAndSetOrdered(false, true)) + QMetaObject::invokeMethod(this, [this]() { + // maybe set m_forceStepMaxPos + + if (isAtEnd()) { + setForceStepDone(); + } + else { + m_explicitNextFrameTime = Clock::now(); + scheduleNextStep(); + } + }); +} + +bool Renderer::isStepForced() const +{ + return m_isStepForced; +} + +void Renderer::setInitialPosition(TimePoint tp, qint64 trackPos) +{ + QMetaObject::invokeMethod(this, [this, tp, trackPos]() { + Q_ASSERT(m_loopIndex == 0); + Q_ASSERT(m_frames.empty()); + + m_loopIndex = 0; + m_lastPosition.storeRelease(trackPos); + m_seekPos.storeRelease(trackPos); + + m_timeController.sync(tp, trackPos); + }); +} + +void Renderer::onFinalFrameReceived() +{ + render({}); +} + +void Renderer::render(Frame frame) +{ + const auto isFrameOutdated = frame.isValid() && frame.absoluteEnd() < seekPosition(); + + if (isFrameOutdated) { + qCDebug(qLcRenderer) << "frame outdated! absEnd:" << frame.absoluteEnd() << "absPts" + << frame.absolutePts() << "seekPos:" << seekPosition(); + emit frameProcessed(frame); + return; + } + + m_frames.enqueue(frame); + + if (m_frames.size() == 1) + scheduleNextStep(); +} + +void Renderer::onPauseChanged() +{ + m_timeController.setPaused(isPaused()); + PlaybackEngineObject::onPauseChanged(); +} + +bool Renderer::canDoNextStep() const +{ + return !m_frames.empty() && (m_isStepForced || PlaybackEngineObject::canDoNextStep()); +} + +float Renderer::playbackRate() const +{ + return m_timeController.playbackRate(); +} + +int Renderer::timerInterval() const +{ + if (m_frames.empty()) + return 0; + + auto calculateInterval = [](const TimePoint &nextTime) { + using namespace std::chrono; + + const auto delay = nextTime - Clock::now(); + return std::max(0, static_cast<int>(duration_cast<milliseconds>(delay).count())); + }; + + if (m_explicitNextFrameTime) + return calculateInterval(*m_explicitNextFrameTime); + + if (m_frames.front().isValid()) + return calculateInterval(m_timeController.timeFromPosition(m_frames.front().absolutePts())); + + if (m_lastFrameEnd > 0) + return calculateInterval(m_timeController.timeFromPosition(m_lastFrameEnd)); + + return 0; +} + +bool Renderer::setForceStepDone() +{ + if (!m_isStepForced.testAndSetOrdered(true, false)) + return false; + + m_explicitNextFrameTime.reset(); + emit forceStepDone(); + return true; +} + +void Renderer::doNextStep() +{ + auto frame = m_frames.front(); + + if (setForceStepDone()) { + // if (frame.isValid() && frame.pts() > m_forceStepMaxPos) { + // scheduleNextStep(false); + // return; + // } + } + + const auto result = renderInternal(frame); + + if (result.done) { + m_explicitNextFrameTime.reset(); + m_frames.dequeue(); + + if (frame.isValid()) { + m_lastPosition.storeRelease(std::max(frame.absolutePts(), lastPosition())); + + // TODO: get rid of m_lastFrameEnd or m_seekPos + m_lastFrameEnd = frame.absoluteEnd(); + m_seekPos.storeRelaxed(m_lastFrameEnd); + + const auto loopIndex = frame.loopOffset().index; + if (m_loopIndex < loopIndex) { + m_loopIndex = loopIndex; + emit loopChanged(id(), frame.loopOffset().pos, m_loopIndex); + } + + emit frameProcessed(frame); + } else { + m_lastPosition.storeRelease(std::max(m_lastFrameEnd, lastPosition())); + } + } else { + m_explicitNextFrameTime = Clock::now() + result.recheckInterval; + } + + setAtEnd(result.done && !frame.isValid()); + + scheduleNextStep(false); +} + +std::chrono::microseconds Renderer::frameDelay(const Frame &frame, TimePoint timePoint) const +{ + return std::chrono::duration_cast<std::chrono::microseconds>( + timePoint - m_timeController.timeFromPosition(frame.absolutePts())); +} + +void Renderer::changeRendererTime(std::chrono::microseconds offset) +{ + const auto now = Clock::now(); + const auto pos = m_timeController.positionFromTime(now); + m_timeController.sync(now + offset, pos); + emit synchronized(id(), now + offset, pos); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegrenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h new file mode 100644 index 000000000..99c5ef1b1 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h @@ -0,0 +1,125 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGRENDERER_P_H +#define QFFMPEGRENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegplaybackengineobject_p.h" +#include "playbackengine/qffmpegtimecontroller_p.h" +#include "playbackengine/qffmpegframe_p.h" + +#include <QtCore/qpointer.h> + +#include <chrono> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class Renderer : public PlaybackEngineObject +{ + Q_OBJECT +public: + using TimePoint = TimeController::TimePoint; + using Clock = TimeController::Clock; + Renderer(const TimeController &tc, const std::chrono::microseconds &seekPosTimeOffset = {}); + + void syncSoft(TimePoint tp, qint64 trackPos); + + qint64 seekPosition() const; + + qint64 lastPosition() const; + + void setPlaybackRate(float rate); + + void doForceStep(); + + bool isStepForced() const; + +public slots: + void setInitialPosition(TimePoint tp, qint64 trackPos); + + void onFinalFrameReceived(); + + void render(Frame); + +signals: + void frameProcessed(Frame); + + void synchronized(Id id, TimePoint tp, qint64 pos); + + void forceStepDone(); + + void loopChanged(Id id, qint64 offset, int index); + +protected: + bool setForceStepDone(); + + void onPauseChanged() override; + + bool canDoNextStep() const override; + + int timerInterval() const override; + + virtual void onPlaybackRateChanged() { } + + struct RenderingResult + { + bool done = true; + std::chrono::microseconds recheckInterval = std::chrono::microseconds(0); + }; + + virtual RenderingResult renderInternal(Frame frame) = 0; + + float playbackRate() const; + + std::chrono::microseconds frameDelay(const Frame &frame, + TimePoint timePoint = Clock::now()) const; + + void changeRendererTime(std::chrono::microseconds offset); + + template<typename Output, typename ChangeHandler> + void setOutputInternal(QPointer<Output> &actual, Output *desired, ChangeHandler &&changeHandler) + { + const auto connectionType = thread() == QThread::currentThread() + ? Qt::AutoConnection + : Qt::BlockingQueuedConnection; + auto doer = [desired, changeHandler, &actual]() { + const auto prev = std::exchange(actual, desired); + if (prev != desired) + changeHandler(prev); + }; + QMetaObject::invokeMethod(this, doer, connectionType); + } + +private: + void doNextStep() override; + +private: + TimeController m_timeController; + qint64 m_lastFrameEnd = 0; + QAtomicInteger<qint64> m_lastPosition = 0; + QAtomicInteger<qint64> m_seekPos = 0; + + int m_loopIndex = 0; + QQueue<Frame> m_frames; + + QAtomicInteger<bool> m_isStepForced = false; + std::optional<TimePoint> m_explicitNextFrameTime; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGRENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp new file mode 100644 index 000000000..2f40c53aa --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp @@ -0,0 +1,240 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegstreamdecoder_p.h" +#include "playbackengine/qffmpegmediadataholder_p.h" +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder"); + +namespace QFFmpeg { + +StreamDecoder::StreamDecoder(const Codec &codec, qint64 absSeekPos) + : m_codec(codec), + m_absSeekPos(absSeekPos), + m_trackType(MediaDataHolder::trackTypeFromMediaType(codec.context()->codec_type)) +{ + qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType + << "absSeekPos:" << absSeekPos; + Q_ASSERT(m_trackType != QPlatformMediaPlayer::NTrackTypes); +} + +StreamDecoder::~StreamDecoder() +{ + avcodec_flush_buffers(m_codec.context()); +} + +void StreamDecoder::onFinalPacketReceived() +{ + decode({}); +} + +void StreamDecoder::setInitialPosition(TimePoint, qint64 trackPos) +{ + m_absSeekPos = trackPos; +} + +void StreamDecoder::decode(Packet packet) +{ + m_packets.enqueue(packet); + + scheduleNextStep(); +} + +void StreamDecoder::doNextStep() +{ + auto packet = m_packets.dequeue(); + + auto decodePacket = [this](Packet packet) { + if (trackType() == QPlatformMediaPlayer::SubtitleStream) + decodeSubtitle(packet); + else + decodeMedia(packet); + }; + + if (packet.isValid() && packet.loopOffset().index != m_offset.index) { + decodePacket({}); + + qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:" << packet.loopOffset().index; + + avcodec_flush_buffers(m_codec.context()); + m_offset = packet.loopOffset(); + } + + decodePacket(packet); + + setAtEnd(!packet.isValid()); + + if (packet.isValid()) + emit packetProcessed(packet); + + scheduleNextStep(false); +} + +QPlatformMediaPlayer::TrackType StreamDecoder::trackType() const +{ + return m_trackType; +} + +qint32 StreamDecoder::maxQueueSize(QPlatformMediaPlayer::TrackType type) +{ + switch (type) { + + case QPlatformMediaPlayer::VideoStream: + return 3; + case QPlatformMediaPlayer::AudioStream: + return 9; + case QPlatformMediaPlayer::SubtitleStream: + return 6; /*main packet and closing packet*/ + default: + Q_UNREACHABLE_RETURN(-1); + } +} + +void StreamDecoder::onFrameProcessed(Frame frame) +{ + if (frame.sourceId() != id()) + return; + + --m_pendingFramesCount; + Q_ASSERT(m_pendingFramesCount >= 0); + + scheduleNextStep(); +} + +bool StreamDecoder::canDoNextStep() const +{ + const qint32 maxCount = maxQueueSize(m_trackType); + + return !m_packets.empty() && m_pendingFramesCount < maxCount + && PlaybackEngineObject::canDoNextStep(); +} + +void StreamDecoder::onFrameFound(Frame frame) +{ + if (frame.isValid() && frame.absoluteEnd() < m_absSeekPos) + return; + + Q_ASSERT(m_pendingFramesCount >= 0); + ++m_pendingFramesCount; + emit requestHandleFrame(frame); +} + +void StreamDecoder::decodeMedia(Packet packet) +{ + auto sendPacketResult = sendAVPacket(packet); + + if (sendPacketResult == AVERROR(EAGAIN)) { + // Doc says: + // AVERROR(EAGAIN): input is not accepted in the current state - user + // must read output with avcodec_receive_frame() (once + // all output is read, the packet should be resent, and + // the call will not fail with EAGAIN). + receiveAVFrames(); + sendPacketResult = sendAVPacket(packet); + + if (sendPacketResult != AVERROR(EAGAIN)) + qWarning() << "Unexpected FFmpeg behavior"; + } + + if (sendPacketResult == 0) + receiveAVFrames(); +} + +int StreamDecoder::sendAVPacket(Packet packet) +{ + return avcodec_send_packet(m_codec.context(), packet.isValid() ? packet.avPacket() : nullptr); +} + +void StreamDecoder::receiveAVFrames() +{ + while (true) { + auto avFrame = makeAVFrame(); + + const auto receiveFrameResult = avcodec_receive_frame(m_codec.context(), avFrame.get()); + + if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) + break; + + if (receiveFrameResult < 0) { + emit error(QMediaPlayer::FormatError, err2str(receiveFrameResult)); + break; + } + + onFrameFound({ m_offset, std::move(avFrame), m_codec, 0, id() }); + } +} + +void StreamDecoder::decodeSubtitle(Packet packet) +{ + if (!packet.isValid()) + return; + // qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" << + // (codec->codec->capabilities & AV_CODEC_CAP_DELAY); + AVSubtitle subtitle; + memset(&subtitle, 0, sizeof(subtitle)); + int gotSubtitle = 0; + + const int res = + avcodec_decode_subtitle2(m_codec.context(), &subtitle, &gotSubtitle, packet.avPacket()); + // qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format << + // Qt::hex << (quint64)subtitle.pts; + if (res < 0 || !gotSubtitle) + return; + + // apparently the timestamps in the AVSubtitle structure are not always filled in + // if they are missing, use the packets pts and duration values instead + qint64 start, end; + if (subtitle.pts == AV_NOPTS_VALUE) { + start = m_codec.toUs(packet.avPacket()->pts); + end = start + m_codec.toUs(packet.avPacket()->duration); + } else { + auto pts = timeStampUs(subtitle.pts, AVRational{ 1, AV_TIME_BASE }); + start = *pts + qint64(subtitle.start_display_time) * 1000; + end = *pts + qint64(subtitle.end_display_time) * 1000; + } + + if (end <= start) { + qWarning() << "Invalid subtitle time"; + return; + } + // qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):"; + QString text; + for (uint i = 0; i < subtitle.num_rects; ++i) { + const auto *r = subtitle.rects[i]; + // qCDebug(qLcDecoder) << " subtitletext:" << r->text << "/" << r->ass; + if (i) + text += QLatin1Char('\n'); + if (r->text) + text += QString::fromUtf8(r->text); + else { + const char *ass = r->ass; + int nCommas = 0; + while (*ass) { + if (nCommas == 8) + break; + if (*ass == ',') + ++nCommas; + ++ass; + } + text += QString::fromUtf8(ass); + } + } + text.replace(QLatin1String("\\N"), QLatin1String("\n")); + text.replace(QLatin1String("\\n"), QLatin1String("\n")); + text.replace(QLatin1String("\r\n"), QLatin1String("\n")); + if (text.endsWith(QLatin1Char('\n'))) + text.chop(1); + + onFrameFound({ m_offset, text, start, end - start, id() }); + + // TODO: maybe optimize + onFrameFound({ m_offset, QString(), end, 0, id() }); +} +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegstreamdecoder_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h new file mode 100644 index 000000000..1acc07983 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h @@ -0,0 +1,87 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGSTREAMDECODER_P_H +#define QFFMPEGSTREAMDECODER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// +#include "playbackengine/qffmpegplaybackengineobject_p.h" +#include "playbackengine/qffmpegframe_p.h" +#include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" +#include "private/qplatformmediaplayer_p.h" + +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class StreamDecoder : public PlaybackEngineObject +{ + Q_OBJECT +public: + StreamDecoder(const Codec &codec, qint64 absSeekPos); + + ~StreamDecoder(); + + QPlatformMediaPlayer::TrackType trackType() const; + + // Maximum number of frames that we are allowed to keep in render queue + static qint32 maxQueueSize(QPlatformMediaPlayer::TrackType type); + +public slots: + void setInitialPosition(TimePoint tp, qint64 trackPos); + + void decode(Packet); + + void onFinalPacketReceived(); + + void onFrameProcessed(Frame frame); + +signals: + void requestHandleFrame(Frame frame); + + void packetProcessed(Packet); + +protected: + bool canDoNextStep() const override; + + void doNextStep() override; + +private: + void decodeMedia(Packet); + + void decodeSubtitle(Packet); + + void onFrameFound(Frame frame); + + int sendAVPacket(Packet); + + void receiveAVFrames(); + +private: + Codec m_codec; + qint64 m_absSeekPos = 0; + const QPlatformMediaPlayer::TrackType m_trackType; + + qint32 m_pendingFramesCount = 0; + + LoopOffset m_offset; + + QQueue<Packet> m_packets; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGSTREAMDECODER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer.cpp new file mode 100644 index 000000000..789c9b53b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer.cpp @@ -0,0 +1,44 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegsubtitlerenderer_p.h" + +#include "qvideosink.h" +#include "qdebug.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +SubtitleRenderer::SubtitleRenderer(const TimeController &tc, QVideoSink *sink) + : Renderer(tc), m_sink(sink) +{ +} + +void SubtitleRenderer::setOutput(QVideoSink *sink, bool cleanPrevSink) +{ + setOutputInternal(m_sink, sink, [cleanPrevSink](QVideoSink *prev) { + if (prev && cleanPrevSink) + prev->setSubtitleText({}); + }); +} + +SubtitleRenderer::~SubtitleRenderer() +{ + if (m_sink) + m_sink->setSubtitleText({}); +} + +Renderer::RenderingResult SubtitleRenderer::renderInternal(Frame frame) +{ + if (m_sink) + m_sink->setSubtitleText(frame.isValid() ? frame.text() : QString()); + + return {}; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegsubtitlerenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer_p.h new file mode 100644 index 000000000..805212e83 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer_p.h @@ -0,0 +1,48 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGSUBTITLERENDERER_P_H +#define QFFMPEGSUBTITLERENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegrenderer_p.h" + +#include <QtCore/qpointer.h> + +QT_BEGIN_NAMESPACE + +class QVideoSink; + +namespace QFFmpeg { + +class SubtitleRenderer : public Renderer +{ + Q_OBJECT +public: + SubtitleRenderer(const TimeController &tc, QVideoSink *sink); + + ~SubtitleRenderer() override; + + void setOutput(QVideoSink *sink, bool cleanPrevSink = false); + +protected: + RenderingResult renderInternal(Frame frame) override; + +private: + QPointer<QVideoSink> m_sink; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGSUBTITLERENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller.cpp new file mode 100644 index 000000000..8352384b4 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller.cpp @@ -0,0 +1,165 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegtimecontroller_p.h" + +#include "qglobal.h" +#include "qdebug.h" + +#include <algorithm> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +TimeController::TimeController() +{ + sync(); +} + +TimeController::PlaybackRate TimeController::playbackRate() const +{ + return m_playbackRate; +} + +void TimeController::setPlaybackRate(PlaybackRate playbackRate) +{ + if (playbackRate == m_playbackRate) + return; + + Q_ASSERT(playbackRate > 0.f); + + scrollTimeTillNow(); + m_playbackRate = playbackRate; + + if (m_softSyncData) + m_softSyncData = makeSoftSyncData(m_timePoint, m_position, m_softSyncData->dstTimePoint); +} + +void TimeController::sync(qint64 trackPos) +{ + sync(Clock::now(), trackPos); +} + +void TimeController::sync(const TimePoint &tp, qint64 pos) +{ + m_softSyncData.reset(); + m_position = TrackTime(pos); + m_timePoint = tp; +} + +void TimeController::syncSoft(const TimePoint &tp, qint64 pos, const Clock::duration &fixingTime) +{ + const auto srcTime = Clock::now(); + const auto srcPos = positionFromTime(srcTime, true); + const auto dstTime = srcTime + fixingTime; + + m_position = TrackTime(pos); + m_timePoint = tp; + + m_softSyncData = makeSoftSyncData(srcTime, TrackTime(srcPos), dstTime); +} + +qint64 TimeController::currentPosition(const Clock::duration &offset) const +{ + return positionFromTime(Clock::now() + offset); +} + +void TimeController::setPaused(bool paused) +{ + if (m_paused == paused) + return; + + scrollTimeTillNow(); + m_paused = paused; +} + +qint64 TimeController::positionFromTime(TimePoint tp, bool ignorePause) const +{ + tp = m_paused && !ignorePause ? m_timePoint : tp; + + if (m_softSyncData && tp < m_softSyncData->dstTimePoint) { + const PlaybackRate rate = + tp > m_softSyncData->srcTimePoint ? m_softSyncData->internalRate : m_playbackRate; + + return (m_softSyncData->srcPosition + + toTrackTime((tp - m_softSyncData->srcTimePoint) * rate)) + .count(); + } + + return positionFromTimeInternal(tp).count(); +} + +TimeController::TimePoint TimeController::timeFromPosition(qint64 pos, bool ignorePause) const +{ + auto position = m_paused && !ignorePause ? m_position : TrackTime(pos); + + if (m_softSyncData && position < m_softSyncData->dstPosition) { + const auto rate = position > m_softSyncData->srcPosition ? m_softSyncData->internalRate + : m_playbackRate; + return m_softSyncData->srcTimePoint + + toClockTime((position - m_softSyncData->srcPosition) / rate); + } + + return timeFromPositionInternal(position); +} + +TimeController::SoftSyncData TimeController::makeSoftSyncData(const TimePoint &srcTp, + const TrackTime &srcPos, + const TimePoint &dstTp) const +{ + SoftSyncData result; + result.srcTimePoint = srcTp; + result.srcPosition = srcPos; + result.dstTimePoint = dstTp; + result.srcPosOffest = srcPos - positionFromTimeInternal(srcTp); + result.dstPosition = positionFromTimeInternal(dstTp); + result.internalRate = + static_cast<PlaybackRate>(toClockTime(TrackTime(result.dstPosition - srcPos)).count()) + / (dstTp - srcTp).count(); + + return result; +} + +TimeController::TrackTime TimeController::positionFromTimeInternal(const TimePoint &tp) const +{ + return m_position + toTrackTime((tp - m_timePoint) * m_playbackRate); +} + +TimeController::TimePoint TimeController::timeFromPositionInternal(const TrackTime &pos) const +{ + return m_timePoint + toClockTime(TrackTime(pos - m_position) / m_playbackRate); +} + +void TimeController::scrollTimeTillNow() +{ + const auto now = Clock::now(); + if (!m_paused) { + m_position = positionFromTimeInternal(now); + + // let's forget outdated syncronizations + if (m_softSyncData && m_softSyncData->dstTimePoint <= now) + m_softSyncData.reset(); + } else if (m_softSyncData) { + m_softSyncData->dstTimePoint += now - m_timePoint; + m_softSyncData->srcTimePoint += now - m_timePoint; + } + + m_timePoint = now; +} + +template<typename T> +TimeController::Clock::duration TimeController::toClockTime(const T &t) +{ + return std::chrono::duration_cast<Clock::duration>(t); +} + +template<typename T> +TimeController::TrackTime TimeController::toTrackTime(const T &t) +{ + return std::chrono::duration_cast<TrackTime>(t); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller_p.h new file mode 100644 index 000000000..93ced7e64 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller_p.h @@ -0,0 +1,94 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGTIMECONTROLLER_P_H +#define QFFMPEGTIMECONTROLLER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qglobal.h" + +#include <chrono> +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class TimeController +{ + using TrackTime = std::chrono::microseconds; + +public: + using Clock = std::chrono::steady_clock; + using TimePoint = Clock::time_point; + using PlaybackRate = float; + + TimeController(); + + PlaybackRate playbackRate() const; + + void setPlaybackRate(PlaybackRate playbackRate); + + void sync(qint64 trackPos = 0); + + void sync(const TimePoint &tp, qint64 pos); + + void syncSoft(const TimePoint &tp, qint64 pos, + const Clock::duration &fixingTime = std::chrono::seconds(4)); + + qint64 currentPosition(const Clock::duration &offset = Clock::duration{ 0 }) const; + + void setPaused(bool paused); + + qint64 positionFromTime(TimePoint tp, bool ignorePause = false) const; + + TimePoint timeFromPosition(qint64 pos, bool ignorePause = false) const; + +private: + struct SoftSyncData + { + TimePoint srcTimePoint; + TrackTime srcPosition; + TimePoint dstTimePoint; + TrackTime srcPosOffest; + TrackTime dstPosition; + PlaybackRate internalRate = 1; + }; + + SoftSyncData makeSoftSyncData(const TimePoint &srcTp, const TrackTime &srcPos, + const TimePoint &dstTp) const; + + TrackTime positionFromTimeInternal(const TimePoint &tp) const; + + TimePoint timeFromPositionInternal(const TrackTime &pos) const; + + void scrollTimeTillNow(); + + template<typename T> + static Clock::duration toClockTime(const T &t); + + template<typename T> + static TrackTime toTrackTime(const T &t); + +private: + bool m_paused = true; + PlaybackRate m_playbackRate = 1; + TrackTime m_position; + TimePoint m_timePoint = {}; + std::optional<SoftSyncData> m_softSyncData; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGTIMECONTROLLER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp new file mode 100644 index 000000000..dceb00f83 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegvideorenderer_p.h" +#include "qffmpegvideobuffer_p.h" +#include "qvideosink.h" +#include "private/qvideoframe_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +VideoRenderer::VideoRenderer(const TimeController &tc, QVideoSink *sink, QtVideo::Rotation rotation) + : Renderer(tc), m_sink(sink), m_rotation(rotation) +{ +} + +void VideoRenderer::setOutput(QVideoSink *sink, bool cleanPrevSink) +{ + setOutputInternal(m_sink, sink, [cleanPrevSink](QVideoSink *prev) { + if (prev && cleanPrevSink) + prev->setVideoFrame({}); + }); +} + +VideoRenderer::RenderingResult VideoRenderer::renderInternal(Frame frame) +{ + if (!m_sink) + return {}; + + if (!frame.isValid()) { + m_sink->setVideoFrame({}); + return {}; + } + + // qCDebug(qLcVideoRenderer) << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi(); + + const auto codec = frame.codec(); + Q_ASSERT(codec); + +#ifdef Q_OS_ANDROID + // QTBUG-108446 + // In general case, just creation of frames context is not correct since + // frames may require additional specific data for hw contexts, so + // just setting of hw_frames_ctx is not enough. + // TODO: investigate the case in order to remove or fix the code. + if (codec->hwAccel() && !frame.avFrame()->hw_frames_ctx) { + HWAccel *hwaccel = codec->hwAccel(); + AVFrame *avframe = frame.avFrame(); + if (!hwaccel->hwFramesContext()) + hwaccel->createFramesContext(AVPixelFormat(avframe->format), + { avframe->width, avframe->height }); + + if (hwaccel->hwFramesContext()) + avframe->hw_frames_ctx = av_buffer_ref(hwaccel->hwFramesContextAsBuffer()); + } +#endif + + const auto pixelAspectRatio = codec->pixelAspectRatio(frame.avFrame()); + auto buffer = std::make_unique<QFFmpegVideoBuffer>(frame.takeAVFrame(), pixelAspectRatio); + QVideoFrameFormat format(buffer->size(), buffer->pixelFormat()); + format.setColorSpace(buffer->colorSpace()); + format.setColorTransfer(buffer->colorTransfer()); + format.setColorRange(buffer->colorRange()); + format.setMaxLuminance(buffer->maxNits()); + format.setRotation(m_rotation); + QVideoFrame videoFrame = QVideoFramePrivate::createFrame(std::move(buffer), format); + videoFrame.setStartTime(frame.pts()); + videoFrame.setEndTime(frame.end()); + m_sink->setVideoFrame(videoFrame); + + return {}; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegvideorenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer_p.h new file mode 100644 index 000000000..4866420e8 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer_p.h @@ -0,0 +1,47 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGVIDEORENDERER_P_H +#define QFFMPEGVIDEORENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegrenderer_p.h" + +#include <QtCore/qpointer.h> + +QT_BEGIN_NAMESPACE + +class QVideoSink; + +namespace QFFmpeg { + +class VideoRenderer : public Renderer +{ + Q_OBJECT +public: + VideoRenderer(const TimeController &tc, QVideoSink *sink, QtVideo::Rotation rotation); + + void setOutput(QVideoSink *sink, bool cleanPrevSink = false); + +protected: + RenderingResult renderInternal(Frame frame) override; + +private: + QPointer<QVideoSink> m_sink; + QtVideo::Rotation m_rotation; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGVIDEORENDERER_P_H |