Improve the criteria of the demuxer's bufferization

AVPacket::duration can be empty and should also rely on the position
the difference when checking stream buffering limitations.

In the previous implementation, we calculated "bufferedDuration" for
each stream as an amount of AVPacket::duration for all packets that
have been got from the ffmpeg demuxer but haven't been processed
(decoded).

In some cases, ffmpeg exposes packets with duration = 0 that break
the mechanism. The CR introduces a new metric
"maxSentPacketPos - maxProcessedPacketPos" that shows approximately
the same. Probably, the previous criteria will be dropped after
additional investigations.

Pick-to: 6.6 6.5
Change-Id: Ic138ea267080a65a969bc77be52e50d3bb96030a
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>

2 files changed, 64 insertions, 22 deletions

diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp
index fba48f3da..5016fb6ef 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp
@@ -7,10 +7,10 @@
 QT_BEGIN_NAMESPACE
 
 // 4 sec for buffering. TODO: maybe move to env var customization
-static constexpr qint64 MaxBufferingTimeUs = 4'000'000;
+static constexpr qint64 MaxBufferedDurationUs = 4'000'000;
 
 // around 4 sec of hdr video
-static constexpr qint64 MaxBufferingSize = 32 * 1024 * 1024;
+static constexpr qint64 MaxBufferedSize = 32 * 1024 * 1024;
 
 namespace QFFmpeg {
 
@@ -24,10 +24,11 @@ static qint64 streamTimeToUs(const AVStream *stream, qint64 time)
     return res ? *res : time;
 }
 
-static auto isStreamLimitReached = [](const auto &streamIndexToData) {
-    return streamIndexToData.second.bufferingTime >= MaxBufferingTimeUs
-            || streamIndexToData.second.bufferingSize >= MaxBufferingSize;
-};
+static qint64 packetEndPos(const AVStream *stream, const Packet &packet)
+{
+    return packet.loopOffset().pos
+            + streamTimeToUs(stream, packet.avPacket()->pts + packet.avPacket()->duration);
+}
 
 Demuxer::Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset,
                  const StreamIndexes &streamIndexes, int loops)
@@ -68,8 +69,8 @@ void Demuxer::doNextStep()
         } else {
             m_seeked = false;
             m_posWithOffset.pos = 0;
-            m_posWithOffset.offset.pos = m_endPts;
-            m_endPts = 0;
+            m_posWithOffset.offset.pos = m_maxPacketsEndPos;
+            m_maxPacketsEndPos = 0;
 
             ensureSeeked();
 
@@ -88,15 +89,20 @@ void Demuxer::doNextStep()
     const auto stream = m_context->streams[streamIndex];
 
     auto it = m_streams.find(streamIndex);
-
     if (it != m_streams.end()) {
-        const auto packetEndPos = streamTimeToUs(stream, avPacket.pts + avPacket.duration);
-        m_endPts = std::max(m_endPts, m_posWithOffset.offset.pos + packetEndPos);
+        auto &streamData = it->second;
+
+        const auto endPos = packetEndPos(stream, packet);
+        m_maxPacketsEndPos = qMax(m_maxPacketsEndPos, endPos);
 
-        it->second.bufferingTime += streamTimeToUs(stream, avPacket.duration);
-        it->second.bufferingSize += avPacket.size;
+        // Increase buffered metrics as the packet has been processed.
 
-        if (!m_buffered && isStreamLimitReached(*it)) {
+        streamData.bufferedDuration += streamTimeToUs(stream, avPacket.duration);
+        streamData.bufferedSize += avPacket.size;
+        streamData.maxSentPacketsPos = qMax(streamData.maxSentPacketsPos, endPos);
+        updateStreamDataLimitFlag(streamData);
+
+        if (!m_buffered && streamData.isDataLimitReached) {
             m_buffered = true;
             emit packetsBuffered();
         }
@@ -124,14 +130,23 @@ void Demuxer::onPacketProcessed(Packet packet)
     auto &avPacket = *packet.avPacket();
 
     const auto streamIndex = avPacket.stream_index;
+    const auto stream = m_context->streams[streamIndex];
     auto it = m_streams.find(streamIndex);
 
     if (it != m_streams.end()) {
-        it->second.bufferingTime -= streamTimeToUs(m_context->streams[streamIndex], avPacket.duration);
-        it->second.bufferingSize -= avPacket.size;
+        auto &streamData = it->second;
+
+        // Decrease buffered metrics as new data (the packet) has been received (buffered)
 
-        Q_ASSERT(it->second.bufferingTime >= 0);
-        Q_ASSERT(it->second.bufferingSize >= 0);
+        streamData.bufferedDuration -= streamTimeToUs(stream, avPacket.duration);
+        streamData.bufferedSize -= avPacket.size;
+        streamData.maxProcessedPacketPos =
+                qMax(streamData.maxProcessedPacketPos, packetEndPos(stream, packet));
+
+        Q_ASSERT(it->second.bufferedDuration >= 0);
+        Q_ASSERT(it->second.bufferedSize >= 0);
+
+        updateStreamDataLimitFlag(streamData);
     }
 
     scheduleNextStep();
@@ -139,8 +154,18 @@ void Demuxer::onPacketProcessed(Packet packet)
 
 bool Demuxer::canDoNextStep() const
 {
+    auto isDataLimitReached = [](const auto &streamIndexToData) {
+        return streamIndexToData.second.isDataLimitReached;
+    };
+
+    // Demuxer waits:
+    //     - if it's paused
+    //     - if the end has been reached
+    //     - if streams are empty (probably, should be handled on the initialization)
+    //     - if at least one of the streams has reached the data limit (duration or size)
+
     return PlaybackEngineObject::canDoNextStep() && !isAtEnd() && !m_streams.empty()
-            && std::none_of(m_streams.begin(), m_streams.end(), isStreamLimitReached);
+            && std::none_of(m_streams.begin(), m_streams.end(), isDataLimitReached);
 }
 
 void Demuxer::ensureSeeked()
@@ -188,6 +213,15 @@ void Demuxer::setLoops(int loopsCount)
     m_loops.storeRelease(loopsCount);
 }
 
+void Demuxer::updateStreamDataLimitFlag(StreamData &streamData)
+{
+    const auto packetsPosDiff = streamData.maxSentPacketsPos - streamData.maxProcessedPacketPos;
+    streamData.isDataLimitReached =
+           streamData.bufferedDuration >= MaxBufferedDurationUs
+        || (streamData.bufferedDuration == 0 && packetsPosDiff >= MaxBufferedDurationUs)
+        || streamData.bufferedSize >= MaxBufferedSize;
+}
+
 } // namespace QFFmpeg
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h
index d2a8e064c..b72056185 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h
@@ -58,16 +58,24 @@ private:
     struct StreamData
     {
         QPlatformMediaPlayer::TrackType trackType = QPlatformMediaPlayer::TrackType::NTrackTypes;
-        qint64 bufferingTime = 0;
-        qint64 bufferingSize = 0;
+        qint64 bufferedDuration = 0;
+        qint64 bufferedSize = 0;
+
+        qint64 maxSentPacketsPos = 0;
+        qint64 maxProcessedPacketPos = 0;
+
+        bool isDataLimitReached = false;
     };
 
+    void updateStreamDataLimitFlag(StreamData &streamData);
+
+private:
     AVFormatContext *m_context = nullptr;
     bool m_seeked = false;
     bool m_firstPacketFound = false;
     std::unordered_map<int, StreamData> m_streams;
     PositionWithOffset m_posWithOffset;
-    qint64 m_endPts = 0;
+    qint64 m_maxPacketsEndPos = 0;
     QAtomicInt m_loops = QMediaPlayer::Once;
     bool m_buffered = false;
 };