Implement a workaround for cases packet.pts < packet.dts

Some ffmpeg video encoders create packets where dts (decoding time)
overtakes pts (presentation time).

Muxer doesn't accept such packets.
Shifting dts down fixes the problem.

Change-Id: Ia27d15650a0d9a8c273229dc4516e18551264676
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
(cherry picked from commit 49bf21047413fab156345b9f788d2332ba8c61dd)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

2 files changed, 49 insertions, 14 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp
index 700d3c2cd..fa596e276 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp
@@ -309,24 +309,55 @@ AVPacketUPtr VideoFrameEncoder::retrievePacket()
 {
     if (!d || !d->codecContext)
         return nullptr;
-    AVPacketUPtr packet(av_packet_alloc());
-    int ret = avcodec_receive_packet(d->codecContext.get(), packet.get());
-    if (ret < 0) {
-        if (ret != AVERROR(EOF) && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
-            qCDebug(qLcVideoFrameEncoder) << "Error receiving packet" << ret << err2str(ret);
-        return nullptr;
-    }
-    auto ts = timeStampMs(packet->pts, d->stream->time_base);
 
-    qCDebug(qLcVideoFrameEncoder) << "got a packet" << packet->pts << packet->dts << (ts ? *ts : 0);
+    auto getPacket = [&]() {
+        AVPacketUPtr packet(av_packet_alloc());
+        const int ret = avcodec_receive_packet(d->codecContext.get(), packet.get());
+        if (ret < 0) {
+            if (ret != AVERROR(EOF) && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
+                qCDebug(qLcVideoFrameEncoder) << "Error receiving packet" << ret << err2str(ret);
+            return AVPacketUPtr{};
+        }
+        auto ts = timeStampMs(packet->pts, d->stream->time_base);
+
+        qCDebug(qLcVideoFrameEncoder)
+                << "got a packet" << packet->pts << packet->dts << (ts ? *ts : 0);
+
+        packet->stream_index = d->stream->id;
+        return packet;
+    };
+
+    auto fixPacketDts = [&](AVPacket &packet) {
+        // Workaround for some ffmpeg codecs bugs (e.g. nvenc)
+        // Ideally, packet->pts < packet->dts is not expected
+
+        if (packet.dts == AV_NOPTS_VALUE)
+            return true;
+
+        packet.dts -= m_packetDtsOffset;
+
+        if (packet.pts != AV_NOPTS_VALUE && packet.pts < packet.dts) {
+            m_packetDtsOffset += packet.dts - packet.pts;
+            packet.dts = packet.pts;
+
+            if (m_prevPacketDts != AV_NOPTS_VALUE && packet.dts < m_prevPacketDts) {
+                qCWarning(qLcVideoFrameEncoder)
+                        << "Skip packet; failed to fix dts:" << packet.dts << m_prevPacketDts;
+                return false;
+            }
+        }
+
+        m_prevPacketDts = packet.dts;
+
+        return true;
+    };
 
-    if (packet->dts != AV_NOPTS_VALUE && packet->pts < packet->dts) {
-        // the case seems to be an ffmpeg bug
-        packet->dts = AV_NOPTS_VALUE;
+    while (auto packet = getPacket()) {
+        if (fixPacketDts(*packet))
+            return packet;
     }
 
-    packet->stream_index = d->stream->id;
-    return packet;
+    return nullptr;
 }
 
 void VideoFrameEncoder::updateConversions()
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h
index c4c2b9c86..6b3bef9ff 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h
@@ -75,6 +75,10 @@ private:
     bool initTargetFormats();
 
     bool initCodecContext(AVFormatContext *formatContext);
+
+private:
+    int64_t m_prevPacketDts = AV_NOPTS_VALUE;
+    int64_t m_packetDtsOffset = 0;
 };