Add rotation property to QVideoFrameFormatHEADdev

We initialize QMediaRecorder by its properties and matching
QVideoFrameFormat of the source. That's why, we need the property in
QVideoFrameFormat for proper initialization.

Note, we already have 'isMirrored' in QVideoFrameFormat,
'mirrored' in QVideoFrame, 'rotation' in QVideoFrame,
so adding rotation to QVdieoFrameFormat looks consistent.

[ChangeLog][QVideoFrameFormat] Added the property 'rotation' to QVideoFrameFormat.

Change-Id: I62abd2e8a1ade4e1e2144c2adc6bc6606828af88
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>

10 files changed, 87 insertions, 12 deletions

diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 9a4d62777..d30017dab 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -683,7 +683,7 @@ void QVideoFrame::setEndTime(qint64 time)
 void QVideoFrame::setRotation(QtVideo::Rotation angle)
 {
     if (d)
-        d->rotation = angle;
+        d->format.setRotation(angle);
 }
 
 /*!
@@ -691,7 +691,7 @@ void QVideoFrame::setRotation(QtVideo::Rotation angle)
  */
 QtVideo::Rotation QVideoFrame::rotation() const
 {
-    return QtVideo::Rotation(d ? d->rotation : QtVideo::Rotation::None);
+    return d ? d->format.rotation() : QtVideo::Rotation::None;
 }
 
 /*!
diff --git a/src/multimedia/video/qvideoframe_p.h b/src/multimedia/video/qvideoframe_p.h
index af891613d..cb8b7b747 100644
--- a/src/multimedia/video/qvideoframe_p.h
+++ b/src/multimedia/video/qvideoframe_p.h
@@ -49,7 +49,6 @@ public:
     int mappedCount = 0;
     QMutex mapMutex;
     QString subtitleText;
-    QtVideo::Rotation rotation = QtVideo::Rotation::None;
     QImage image;
     QMutex imageMutex;
 
diff --git a/src/multimedia/video/qvideoframeformat.cpp b/src/multimedia/video/qvideoframeformat.cpp
index 63388aab6..b3177234f 100644
--- a/src/multimedia/video/qvideoframeformat.cpp
+++ b/src/multimedia/video/qvideoframeformat.cpp
@@ -39,7 +39,8 @@ public:
             && viewport == other.viewport
             && frameRatesEqual(frameRate, other.frameRate)
             && colorSpace == other.colorSpace
-            && mirrored == other.mirrored)
+            && mirrored == other.mirrored
+            && rotation == other.rotation)
             return true;
 
         return false;
@@ -60,6 +61,7 @@ public:
     float frameRate = 0.0;
     float maxLuminance = -1.;
     bool mirrored = false;
+    QtVideo::Rotation rotation = QtVideo::Rotation::None;
 };
 
 QT_DEFINE_QESDP_SPECIALIZATION_DTOR(QVideoFrameFormatPrivate);
@@ -683,6 +685,23 @@ void QVideoFrameFormat::setMirrored(bool mirrored)
 }
 
 /*!
+    Returns the rotation angle the matching video frame should be rotated clockwise before displaying.
+ */
+QtVideo::Rotation QVideoFrameFormat::rotation() const
+{
+    return d->rotation;
+}
+
+/*!
+    Sets the \a rotation angle the matching video frame should be rotated clockwise before displaying.
+ */
+void QVideoFrameFormat::setRotation(QtVideo::Rotation rotation)
+{
+    detach();
+    d->rotation = rotation;
+}
+
+/*!
     \internal
 */
 QString QVideoFrameFormat::vertexShaderFileName() const
diff --git a/src/multimedia/video/qvideoframeformat.h b/src/multimedia/video/qvideoframeformat.h
index 9d74e9695..18dc9952d 100644
--- a/src/multimedia/video/qvideoframeformat.h
+++ b/src/multimedia/video/qvideoframeformat.h
@@ -5,6 +5,7 @@
 #define QVIDEOSURFACEFORMAT_H
 
 #include <QtMultimedia/qtmultimediaglobal.h>
+#include <QtMultimedia/qtvideo.h>
 
 #include <QtCore/qlist.h>
 #include <QtCore/qmetatype.h>
@@ -182,6 +183,9 @@ public:
     bool isMirrored() const;
     void setMirrored(bool mirrored);
 
+    QtVideo::Rotation rotation() const;
+    void setRotation(QtVideo::Rotation rotation);
+
     QString vertexShaderFileName() const;
     QString fragmentShaderFileName() const;
     void updateUniformData(QByteArray *dst, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity) const;
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp
index 7c8e90552..72b13064c 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp
@@ -62,10 +62,10 @@ VideoRenderer::RenderingResult VideoRenderer::renderInternal(Frame frame)
     format.setColorTransfer(buffer->colorTransfer());
     format.setColorRange(buffer->colorRange());
     format.setMaxLuminance(buffer->maxNits());
+    format.setRotation(m_rotation);
     QVideoFrame videoFrame(buffer.release(), format);
     videoFrame.setStartTime(frame.pts());
     videoFrame.setEndTime(frame.end());
-    videoFrame.setRotation(m_rotation);
     m_sink->setVideoFrame(videoFrame);
 
     return {};
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeg.cpp b/src/plugins/multimedia/ffmpeg/qffmpeg.cpp
index fe86e480f..e998495d8 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeg.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeg.cpp
@@ -509,6 +509,32 @@ AVPixelFormat pixelFormatForHwDevice(AVHWDeviceType deviceType)
     }
 }
 
+AVPacketSideData *addStreamSideData(AVStream *stream, AVPacketSideData sideData)
+{
+    QScopeGuard freeData([&sideData]() { av_free(sideData.data); });
+#if QT_FFMPEG_STREAM_SIDE_DATA_DEPRECATED
+    AVPacketSideData *result = av_packet_side_data_add(
+                                          &stream->codecpar->coded_side_data,
+                                          &stream->codecpar->nb_coded_side_data,
+                                          sideData.type,
+                                          sideData.data,
+                                          sideData.size,
+                                          0);
+    if (result) {
+        // If the result is not null, the ownership is taken by AVStream,
+        // otherwise the data must be deleted.
+        freeData.dismiss();
+        return result;
+    }
+#else
+    Q_UNUSED(stream);
+    // TODO: implement for older FFmpeg versions
+    qWarning() << "Adding stream side data is not supported for FFmpeg < 6.1";
+#endif
+
+    return nullptr;
+}
+
 const AVPacketSideData *streamSideData(const AVStream *stream, AVPacketSideDataType type)
 {
     Q_ASSERT(stream);
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeg_p.h b/src/plugins/multimedia/ffmpeg/qffmpeg_p.h
index 601b44ccb..6e19b9650 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeg_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpeg_p.h
@@ -227,6 +227,8 @@ void applyExperimentalCodecOptions(const AVCodec *codec, AVDictionary** opts);
 
 AVPixelFormat pixelFormatForHwDevice(AVHWDeviceType deviceType);
 
+AVPacketSideData *addStreamSideData(AVStream *stream, AVPacketSideData sideData);
+
 const AVPacketSideData *streamSideData(const AVStream *stream, AVPacketSideDataType type);
 
 SwrContextUPtr createResampleContext(const AVAudioFormat &inputFormat,
diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp
index d57415e57..5f9dd83c4 100644
--- a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp
@@ -30,9 +30,13 @@ VideoEncoder::VideoEncoder(RecordingEngine &recordingEngine, const QMediaEncoder
         frameRate = 30.;
     }
 
-    m_frameEncoder =
-            VideoFrameEncoder::create(settings, format.frameSize(), frameRate, ffmpegPixelFormat,
-                                      swFormat, recordingEngine.avFormatContext());
+    m_frameEncoder = VideoFrameEncoder::create(settings,
+                                               format.frameSize(),
+                                               format.rotation(),
+                                               frameRate,
+                                               ffmpegPixelFormat,
+                                               swFormat,
+                                               recordingEngine.avFormatContext());
 }
 
 VideoEncoder::~VideoEncoder() = default;
diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp
index f5c63f3ae..18d9c1401 100644
--- a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp
@@ -7,6 +7,10 @@
 #include "qffmpegvideoencoderutils_p.h"
 #include <qloggingcategory.h>
 
+extern "C" {
+#include "libavutil/display.h"
+}
+
 QT_BEGIN_NAMESPACE
 
 static Q_LOGGING_CATEGORY(qLcVideoFrameEncoder, "qt.multimedia.ffmpeg.videoencoder");
@@ -14,9 +18,13 @@ static Q_LOGGING_CATEGORY(qLcVideoFrameEncoder, "qt.multimedia.ffmpeg.videoencod
 namespace QFFmpeg {
 
 std::unique_ptr<VideoFrameEncoder>
-VideoFrameEncoder::create(const QMediaEncoderSettings &encoderSettings, const QSize &sourceSize,
-                          qreal sourceFrameRate, AVPixelFormat sourceFormat,
-                          AVPixelFormat sourceSWFormat, AVFormatContext *formatContext)
+VideoFrameEncoder::create(const QMediaEncoderSettings &encoderSettings,
+                          const QSize &sourceSize,
+                          QtVideo::Rotation sourceRotation,
+                          qreal sourceFrameRate,
+                          AVPixelFormat sourceFormat,
+                          AVPixelFormat sourceSWFormat,
+                          AVFormatContext *formatContext)
 {
     Q_ASSERT(isSwPixelFormat(sourceSWFormat));
     Q_ASSERT(isHwPixelFormat(sourceFormat) || sourceSWFormat == sourceFormat);
@@ -26,6 +34,7 @@ VideoFrameEncoder::create(const QMediaEncoderSettings &encoderSettings, const QS
     result->m_settings = encoderSettings;
     result->m_sourceSize = sourceSize;
     result->m_sourceFormat = sourceFormat;
+    result->m_sourceRotation = sourceRotation;
 
     // Temporary: check isSwPixelFormat because of android issue (QTBUG-116836)
     result->m_sourceSWFormat = isSwPixelFormat(sourceFormat) ? sourceFormat : sourceSWFormat;
@@ -154,6 +163,15 @@ bool QFFmpeg::VideoFrameEncoder::initCodecContext(AVFormatContext *formatContext
     m_stream->codecpar->height = resolution.height();
     m_stream->codecpar->sample_aspect_ratio = AVRational{ 1, 1 };
 
+    if (m_sourceRotation != QtVideo::Rotation::None) {
+        constexpr auto displayMatrixSize = sizeof(int32_t) * 9;
+        AVPacketSideData sideData = { reinterpret_cast<uint8_t *>(av_malloc(displayMatrixSize)),
+                                      displayMatrixSize, AV_PKT_DATA_DISPLAYMATRIX };
+        av_display_rotation_set(reinterpret_cast<int32_t *>(sideData.data),
+                                static_cast<double>(m_sourceRotation));
+        addStreamSideData(m_stream, sideData);
+    }
+
     Q_ASSERT(m_codec);
 
     m_stream->time_base = adjustFrameTimeBase(m_codec->supported_framerates, m_codecFrameRate);
diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h
index b44e9cbf7..b8e51e6c1 100644
--- a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h
+++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h
@@ -27,7 +27,9 @@ class VideoFrameEncoder
 {
 public:
     static std::unique_ptr<VideoFrameEncoder> create(const QMediaEncoderSettings &encoderSettings,
-                                                     const QSize &sourceSize, qreal sourceFrameRate,
+                                                     const QSize &sourceSize,
+                                                     QtVideo::Rotation sourceRotation,
+                                                     qreal sourceFrameRate,
                                                      AVPixelFormat sourceFormat,
                                                      AVPixelFormat sourceSWFormat,
                                                      AVFormatContext *formatContext);
@@ -60,6 +62,7 @@ private:
 private:
     QMediaEncoderSettings m_settings;
     QSize m_sourceSize;
+    QtVideo::Rotation m_sourceRotation = QtVideo::Rotation::None;
 
     std::unique_ptr<HWAccel> m_accel;
     const AVCodec *m_codec = nullptr;