Prevent possible ffmpeg crash when media header writing failed

We shouldn't write a trailer if the header has got failed.

Change-Id: I6ba7469196682123a3191963351a236fd9abdbb5
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
(cherry picked from commit ed5e09af3f99ef252f17955bf94b2df4c529124c)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

2 files changed, 13 insertions, 4 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp
index 726ea8694..331f9c4fd 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp
@@ -115,6 +115,8 @@ void Encoder::start()
 
     formatContext->metadata = QFFmpegMetaData::toAVMetaData(metaData);
 
+    Q_ASSERT(!isHeaderWritten);
+
     int res = avformat_write_header(formatContext, nullptr);
     if (res < 0) {
         qWarning() << "could not write header, error:" << res << err2str(res);
@@ -122,6 +124,8 @@ void Encoder::start()
         return;
     }
 
+    isHeaderWritten = true;
+
     qCDebug(qLcFFmpegEncoder) << "stream header is successfully written";
 
     muxer->start();
@@ -144,12 +148,15 @@ void EncodingFinalizer::run()
         videoEncoder->kill();
     encoder->muxer->kill();
 
-    int res = av_write_trailer(encoder->formatContext);
-    if (res < 0)
-        qWarning() << "could not write trailer" << res;
+    if (encoder->isHeaderWritten) {
+        const int res = av_write_trailer(encoder->formatContext);
+        if (res < 0)
+            qWarning() << "could not write trailer" << res;
+    }
+    // else ffmpeg might crash
 
     // Close the AVIOContext and release any file handles
-    res = avio_close(encoder->formatContext->pb);
+    const int res = avio_close(encoder->formatContext->pb);
     Q_ASSERT(res == 0);
 
     avformat_free_context(encoder->formatContext);
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h
index 517143b0f..0cf9b18f8 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h
@@ -100,6 +100,8 @@ private:
 
     QMutex timeMutex;
     qint64 timeRecorded = 0;
+
+    bool isHeaderWritten = false;
 };