1 files changed, 104 insertions, 36 deletions

diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp
index 5f9dd83c4..27706580b 100644
--- a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp
@@ -5,13 +5,15 @@
 #include "qffmpegvideobuffer_p.h"
 #include "qffmpegrecordingengine_p.h"
 #include "qffmpegvideoframeencoder_p.h"
+#include "qffmpegrecordingengineutils_p.h"
+#include "private/qvideoframe_p.h"
 #include <QtCore/qloggingcategory.h>
 
 QT_BEGIN_NAMESPACE
 
 namespace QFFmpeg {
 
-static Q_LOGGING_CATEGORY(qLcFFmpegVideoEncoder, "qt.multimedia.ffmpeg.videoencoder");
+Q_STATIC_LOGGING_CATEGORY(qLcFFmpegVideoEncoder, "qt.multimedia.ffmpeg.videoencoder");
 
 VideoEncoder::VideoEncoder(RecordingEngine &recordingEngine, const QMediaEncoderSettings &settings,
                            const QVideoFrameFormat &format, std::optional<AVPixelFormat> hwFormat)
@@ -19,10 +21,8 @@ VideoEncoder::VideoEncoder(RecordingEngine &recordingEngine, const QMediaEncoder
 {
     setObjectName(QLatin1String("VideoEncoder"));
 
-    AVPixelFormat swFormat = QFFmpegVideoBuffer::toAVPixelFormat(format.pixelFormat());
-    AVPixelFormat ffmpegPixelFormat =
-            hwFormat && *hwFormat != AV_PIX_FMT_NONE ? *hwFormat : swFormat;
-    auto frameRate = format.streamFrameRate();
+    const AVPixelFormat swFormat = QFFmpegVideoBuffer::toAVPixelFormat(format.pixelFormat());
+    qreal frameRate = format.streamFrameRate();
     if (frameRate <= 0.) {
         qWarning() << "Invalid frameRate" << frameRate << "; Using the default instead";
 
@@ -30,13 +30,20 @@ VideoEncoder::VideoEncoder(RecordingEngine &recordingEngine, const QMediaEncoder
         frameRate = 30.;
     }
 
-    m_frameEncoder = VideoFrameEncoder::create(settings,
-                                               format.frameSize(),
-                                               format.rotation(),
-                                               frameRate,
-                                               ffmpegPixelFormat,
-                                               swFormat,
-                                               recordingEngine.avFormatContext());
+    VideoFrameEncoder::SourceParams sourceParams;
+    sourceParams.size = format.frameSize();
+    sourceParams.format = hwFormat && *hwFormat != AV_PIX_FMT_NONE ? *hwFormat : swFormat;
+    sourceParams.swFormat = swFormat;
+    sourceParams.rotation = format.rotation();
+    sourceParams.xMirrored = format.isMirrored();
+    sourceParams.yMirrored = format.scanLineDirection() == QVideoFrameFormat::BottomToTop;
+    sourceParams.frameRate = frameRate;
+    sourceParams.colorTransfer = QFFmpeg::toAvColorTransfer(format.colorTransfer());
+    sourceParams.colorSpace = QFFmpeg::toAvColorSpace(format.colorSpace());
+    sourceParams.colorRange = QFFmpeg::toAvColorRange(format.colorRange());
+
+    m_frameEncoder =
+            VideoFrameEncoder::create(settings, sourceParams, recordingEngine.avFormatContext());
 }
 
 VideoEncoder::~VideoEncoder() = default;
@@ -48,25 +55,40 @@ bool VideoEncoder::isValid() const
 
 void VideoEncoder::addFrame(const QVideoFrame &frame)
 {
-    QMutexLocker locker = lockLoopData();
+    if (!frame.isValid()) {
+        setEndOfSourceStream();
+        return;
+    }
+
+    {
+        auto guard = lockLoopData();
 
-    // Drop frames if encoder can not keep up with the video source data rate
-    const bool queueFull = m_videoFrameQueue.size() >= m_maxQueueSize;
+        resetEndOfSourceStream();
 
-    if (queueFull) {
-        qCDebug(qLcFFmpegVideoEncoder) << "RecordingEngine frame queue full. Frame lost.";
-    } else if (!m_paused.loadRelaxed()) {
-        m_videoFrameQueue.push(frame);
+        if (m_paused) {
+            m_shouldAdjustTimeBaseForNextFrame = true;
+            return;
+        }
+
+        // Drop frames if encoder can not keep up with the video source data rate;
+        // canPushFrame might be used instead
+        const bool queueFull = m_videoFrameQueue.size() >= m_maxQueueSize;
 
-        locker.unlock(); // Avoid context switch on wake wake-up
+        if (queueFull) {
+            qCDebug(qLcFFmpegVideoEncoder) << "RecordingEngine frame queue full. Frame lost.";
+            return;
+        }
 
-        dataReady();
+        m_videoFrameQueue.push({ frame, m_shouldAdjustTimeBaseForNextFrame });
+        m_shouldAdjustTimeBaseForNextFrame = false;
     }
+
+    dataReady();
 }
 
-QVideoFrame VideoEncoder::takeFrame()
+VideoEncoder::FrameInfo VideoEncoder::takeFrame()
 {
-    QMutexLocker locker = lockLoopData();
+    auto guard = lockLoopData();
     return dequeueIfPossible(m_videoFrameQueue);
 }
 
@@ -80,10 +102,13 @@ void VideoEncoder::retrievePackets()
 
 void VideoEncoder::init()
 {
+    Q_ASSERT(isValid());
+
     qCDebug(qLcFFmpegVideoEncoder) << "VideoEncoder::init started video device thread.";
     bool ok = m_frameEncoder->open();
     if (!ok)
-        emit m_recordingEngine.error(QMediaRecorder::ResourceError, "Could not initialize encoder");
+        emit m_recordingEngine.sessionError(QMediaRecorder::ResourceError,
+                                            "Could not initialize encoder");
 }
 
 void VideoEncoder::cleanup()
@@ -117,9 +142,9 @@ void VideoEncoder::processOne()
 {
     retrievePackets();
 
-    auto frame = takeFrame();
-    if (!frame.isValid())
-        return;
+    FrameInfo frameInfo = takeFrame();
+    QVideoFrame &frame = frameInfo.frame;
+    Q_ASSERT(frame.isValid());
 
     if (!isValid())
         return;
@@ -128,7 +153,7 @@ void VideoEncoder::processOne()
 
     AVFrameUPtr avFrame;
 
-    auto *videoBuffer = dynamic_cast<QFFmpegVideoBuffer *>(frame.videoBuffer());
+    auto *videoBuffer = dynamic_cast<QFFmpegVideoBuffer *>(QVideoFramePrivate::hwBuffer(frame));
     if (videoBuffer) {
         // ffmpeg video buffer, let's use the native AVFrame stored in there
         auto *hwFrame = videoBuffer->getHWFrame();
@@ -137,7 +162,7 @@ void VideoEncoder::processOne()
     }
 
     if (!avFrame) {
-        frame.map(QVideoFrame::ReadOnly);
+        frame.map(QtVideo::MapMode::ReadOnly);
         auto size = frame.size();
         avFrame = makeAVFrame();
         avFrame->format = m_frameEncoder->sourceFormat();
@@ -149,6 +174,15 @@ void VideoEncoder::processOne()
             avFrame->linesize[i] = frame.bytesPerLine(i);
         }
 
+        // TODO: investigate if we need to set color params to AVFrame.
+        //       Setting only codec carameters might be sufficient.
+        //       What happens if frame color params are set and not equal codec prms?
+        //
+        // QVideoFrameFormat format = frame.surfaceFormat();
+        // avFrame->color_trc = QFFmpeg::toAvColorTransfer(format.colorTransfer());
+        // avFrame->colorspace = QFFmpeg::toAvColorSpace(format.colorSpace());
+        // avFrame->color_range = QFFmpeg::toAvColorRange(format.colorRange());
+
         QImage img;
         if (frame.pixelFormat() == QVideoFrameFormat::Format_Jpeg) {
             // the QImage is cached inside the video frame, so we can take the pointer to the image
@@ -164,14 +198,16 @@ void VideoEncoder::processOne()
                                                new QVideoFrameHolder{ frame, img }, 0);
     }
 
-    if (m_baseTime.loadAcquire() == std::numeric_limits<qint64>::min()) {
-        m_baseTime.storeRelease(frame.startTime() - m_lastFrameTime);
-        qCDebug(qLcFFmpegVideoEncoder) << ">>>> adjusting base time to" << m_baseTime.loadAcquire()
-                                       << frame.startTime() << m_lastFrameTime;
+    const auto [startTime, endTime] = frameTimeStamps(frame);
+
+    if (frameInfo.shouldAdjustTimeBase) {
+        m_baseTime += startTime - m_lastFrameTime;
+        qCDebug(qLcFFmpegVideoEncoder)
+                << ">>>> adjusting base time to" << m_baseTime << startTime << m_lastFrameTime;
     }
 
-    qint64 time = frame.startTime() - m_baseTime.loadAcquire();
-    m_lastFrameTime = frame.endTime() - m_baseTime.loadAcquire();
+    const qint64 time = startTime - m_baseTime;
+    m_lastFrameTime = endTime;
 
     setAVFrameTime(*avFrame, m_frameEncoder->getPts(time), m_frameEncoder->getTimeBase());
 
@@ -182,8 +218,40 @@ void VideoEncoder::processOne()
     int ret = m_frameEncoder->sendFrame(std::move(avFrame));
     if (ret < 0) {
         qCDebug(qLcFFmpegVideoEncoder) << "error sending frame" << ret << err2str(ret);
-        emit m_recordingEngine.error(QMediaRecorder::ResourceError, err2str(ret));
+        emit m_recordingEngine.sessionError(QMediaRecorder::ResourceError, err2str(ret));
+    }
+}
+
+bool VideoEncoder::checkIfCanPushFrame() const
+{
+    if (isRunning())
+        return m_videoFrameQueue.size() < m_maxQueueSize;
+    if (!isFinished())
+        return m_videoFrameQueue.empty();
+
+    return false;
+}
+
+std::pair<qint64, qint64> VideoEncoder::frameTimeStamps(const QVideoFrame &frame) const
+{
+    qint64 startTime = frame.startTime();
+    qint64 endTime = frame.endTime();
+
+    if (startTime == -1) {
+        startTime = m_lastFrameTime;
+        endTime = -1;
+    }
+
+    if (endTime == -1) {
+        qreal frameRate = frame.streamFrameRate();
+        if (frameRate <= 0.)
+            frameRate = m_frameEncoder->settings().videoFrameRate();
+
+        Q_ASSERT(frameRate > 0.f);
+        endTime = startTime + static_cast<qint64>(std::round(VideoFrameTimeBase / frameRate));
     }
+
+    return { startTime, endTime };
 }
 
 } // namespace QFFmpeg