Fix size of QVideoSink when the video is rotated

The size of QVideoSink impacts on sizeHint of QVideoWidget,
and it can be taken by users for their own size hints.

All renderers draw rotated videos like this: if the frame size is
AxB, and the rotation is 90 degrees, the rendering size becomes BxA.

Let's keep QVideoSink::videoSize consistent.

Added a function to qtmm utilities to get rotates size.

Pick-to: 6.5
Change-Id: Ia0cb6f43a598820007fac8178104631a80d96dfb
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
(cherry picked from commit 721cd63b7d8600b2f2801a32e047f6ce24b47e51)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit 93e52743d53eb73d04bce6c5808f57556e8b79e0)

10 files changed, 106 insertions, 26 deletions

diff --git a/src/multimedia/platform/qplatformvideosink.cpp b/src/multimedia/platform/qplatformvideosink.cpp
index 268291da2..abf82af0f 100644
--- a/src/multimedia/platform/qplatformvideosink.cpp
+++ b/src/multimedia/platform/qplatformvideosink.cpp
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include "qplatformvideosink_p.h"
+#include "qmultimediautils_p.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -36,10 +37,14 @@ void QPlatformVideoSink::setVideoFrame(const QVideoFrame &frame)
             return;
         m_currentVideoFrame = frame;
         m_currentVideoFrame.setSubtitleText(m_subtitleText);
-        sizeChanged = m_nativeSize != frame.size();
-        m_nativeSize = frame.size();
+        const auto size = qRotatedFrameSize(frame);
+        if (size != m_nativeSize) {
+            m_nativeSize = size;
+            sizeChanged = true;
+        }
     }
 
+    // emit signals outside the mutex to avoid deadlocks on the user side
     if (sizeChanged)
         emit m_sink->videoSizeChanged();
     emit m_sink->videoFrameChanged(frame);
diff --git a/src/multimedia/qmultimediautils.cpp b/src/multimedia/qmultimediautils.cpp
index 26b71cad6..1b1f59862 100644
--- a/src/multimedia/qmultimediautils.cpp
+++ b/src/multimedia/qmultimediautils.cpp
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include "qmultimediautils_p.h"
+#include "qvideoframe.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -48,4 +49,15 @@ QSize qCalculateFrameSize(QSize resolution, Fraction par)
     return { resolution.width(), resolution.height() * par.denominator / par.numerator };
 }
 
+QSize qRotatedFrameSize(QSize size, int rotation)
+{
+    Q_ASSERT(rotation % 90 == 0);
+    return rotation % 180 ? size.transposed() : size;
+}
+
+QSize qRotatedFrameSize(const QVideoFrame &frame)
+{
+    return qRotatedFrameSize(frame.size(), frame.rotationAngle());
+}
+
 QT_END_NAMESPACE
diff --git a/src/multimedia/qmultimediautils_p.h b/src/multimedia/qmultimediautils_p.h
index 5c1286f85..d4f92a812 100644
--- a/src/multimedia/qmultimediautils_p.h
+++ b/src/multimedia/qmultimediautils_p.h
@@ -16,6 +16,7 @@
 //
 
 #include <QtMultimedia/qtmultimediaglobal.h>
+#include <QtMultimedia/private/qtvideo_p.h>
 #include <QtCore/private/qglobal_p.h>
 #include <qstring.h>
 #include <qsize.h>
@@ -24,6 +25,8 @@
 
 QT_BEGIN_NAMESPACE
 
+class QVideoFrame;
+
 struct QUnexpect
 {
 };
@@ -101,6 +104,17 @@ Q_MULTIMEDIA_EXPORT Fraction qRealToFraction(qreal value);
 
 Q_MULTIMEDIA_EXPORT QSize qCalculateFrameSize(QSize resolution, Fraction pixelAspectRatio);
 
+// TODO: after adding pixel aspect ratio to QVideoFrameFormat, the function should
+// consider PAR as well as rotation
+Q_MULTIMEDIA_EXPORT QSize qRotatedFrameSize(QSize size, int rotation);
+
+inline QSize qRotatedFrameSize(QSize size, QtVideo::Rotation rotation)
+{
+    return qRotatedFrameSize(size, qToUnderlying(rotation));
+}
+
+Q_MULTIMEDIA_EXPORT QSize qRotatedFrameSize(const QVideoFrame &frame);
+
 QT_END_NAMESPACE
 
 #endif // QMULTIMEDIAUTILS_P_H
diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 4b14e9684..348baf023 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -5,6 +5,7 @@
 
 #include "qvideoframe_p.h"
 #include "qvideotexturehelper_p.h"
+#include "qmultimediautils_p.h"
 #include "qmemoryvideobuffer_p.h"
 #include "qvideoframeconverter_p.h"
 #include "qpainter.h"
@@ -689,9 +690,7 @@ void QVideoFrame::paint(QPainter *painter, const QRectF &rect, const PaintOption
     }
 
     QRectF targetRect = rect;
-    QSizeF size = this->size();
-    if (qToUnderlying(rotationAngle()) % 180)
-        size.transpose();
+    QSizeF size = qRotatedFrameSize(*this);
 
     size.scale(targetRect.size(), options.aspectRatioMode);
 
diff --git a/src/multimedia/video/qvideoframeconverter.cpp b/src/multimedia/video/qvideoframeconverter.cpp
index b068555e8..82e0a0af5 100644
--- a/src/multimedia/video/qvideoframeconverter.cpp
+++ b/src/multimedia/video/qvideoframeconverter.cpp
@@ -5,6 +5,7 @@
 #include "qvideoframeconversionhelper_p.h"
 #include "qvideoframeformat.h"
 #include "qvideoframe_p.h"
+#include "qmultimediautils_p.h"
 
 #include <QtCore/qcoreapplication.h>
 #include <QtCore/qsize.h>
@@ -322,11 +323,7 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QtVideo::Rotation rotation
 
     // Do conversion using shaders
 
-    const int rotationIndex = (qToUnderlying(rotation) / 90) % 4;
-
-    QSize frameSize = frame.size();
-    if (rotationIndex % 2)
-        frameSize.transpose();
+    const QSize frameSize = qRotatedFrameSize(frame.size(), rotation);
 
     vertexBuffer.reset(rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_quad)));
     vertexBuffer->create();
@@ -394,7 +391,8 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QtVideo::Rotation rotation
     cb->setViewport({ 0, 0, float(frameSize.width()), float(frameSize.height()) });
     cb->setShaderResources(shaderResourceBindings.get());
 
-    quint32 vertexOffset = quint32(sizeof(float)) * 16 * rotationIndex;
+    const int rotationIndex = (qToUnderlying(rotation) / 90) % 4;
+    const quint32 vertexOffset = quint32(sizeof(float)) * 16 * rotationIndex;
     const QRhiCommandBuffer::VertexInput vbufBinding(vertexBuffer.get(), vertexOffset);
     cb->setVertexInput(0, 1, &vbufBinding);
     cb->draw(4);
diff --git a/src/multimediaquick/qquickvideooutput.cpp b/src/multimediaquick/qquickvideooutput.cpp
index 1a5abccb6..5c45d74ca 100644
--- a/src/multimediaquick/qquickvideooutput.cpp
+++ b/src/multimediaquick/qquickvideooutput.cpp
@@ -11,6 +11,7 @@
 #include <qvideosink.h>
 #include <QtQuick/QQuickWindow>
 #include <private/qquickwindow_p.h>
+#include <private/qmultimediautils_p.h>
 #include <qsgvideonode_p.h>
 
 QT_BEGIN_NAMESPACE
@@ -169,9 +170,7 @@ void QQuickVideoOutput::_q_newFrame(QSize size)
 {
     update();
 
-    if (!qIsDefaultAspect(m_orientation + m_frameOrientation)) {
-        size.transpose();
-    }
+    size = qRotatedFrameSize(size, m_orientation + m_frameOrientation);
 
     if (m_nativeSize != size) {
         m_nativeSize = size;
diff --git a/src/multimediaquick/qsgvideonode_p.cpp b/src/multimediaquick/qsgvideonode_p.cpp
index d89257f72..8c5496cdd 100644
--- a/src/multimediaquick/qsgvideonode_p.cpp
+++ b/src/multimediaquick/qsgvideonode_p.cpp
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include "qsgvideonode_p.h"
+#include "private/qmultimediautils_p.h"
 #include <QtQuick/qsgmaterial.h>
 #include "qsgvideotexture_p.h"
 #include <QtMultimedia/private/qvideotexturehelper_p.h>
@@ -236,8 +237,9 @@ void QSGVideoNode::updateSubtitle(const QVideoFrame &frame)
     QSize subtitleFrameSize = m_rect.size().toSize();
     if (subtitleFrameSize.isEmpty())
         return;
-    if (m_orientation % 180)
-        subtitleFrameSize.transpose();
+
+    subtitleFrameSize = qRotatedFrameSize(subtitleFrameSize, m_orientation);
+
     if (!m_subtitleLayout.update(subtitleFrameSize, frame.subtitleText()))
         return;
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp
index f164104c8..811feb0d5 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp
@@ -612,11 +612,14 @@ void PlaybackEngine::updateVideoSinkSize(QVideoSink *prevSink)
         if (streamIndex >= 0) {
             const auto context = m_media.avContext();
             const auto stream = context->streams[streamIndex];
-            const auto pixelAspectRatio = av_guess_sample_aspect_ratio(context, stream, nullptr);
+            const AVRational pixelAspectRatio =
+                    av_guess_sample_aspect_ratio(context, stream, nullptr);
             // auto size = metaData().value(QMediaMetaData::Resolution)
-            platformVideoSink->setNativeSize(
+            const QSize size =
                     qCalculateFrameSize({ stream->codecpar->width, stream->codecpar->height },
-                                        { pixelAspectRatio.num, pixelAspectRatio.den }));
+                                        { pixelAspectRatio.num, pixelAspectRatio.den });
+
+            platformVideoSink->setNativeSize(qRotatedFrameSize(size, m_media.rotation()));
         }
     }
 }
diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
index 42a03e0b5..e107a5c1a 100644
--- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
+++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
@@ -2846,10 +2846,29 @@ void tst_QMediaPlayerBackend::play_playsRotatedVideoOutput_whenVideoFileHasOrien
     QTest::addColumn<MaybeUrl>("fileURL");
     QTest::addColumn<QRgb>("expectedColor");
     QTest::addColumn<QtVideo::Rotation>("expectedRotationAngle");
-    QTest::addRow("without rotation") << m_colorMatrixVideo << QRgb(0xff0000) << QtVideo::Rotation::None;
-    QTest::addRow("90 deg clockwise") << m_colorMatrix90degClockwiseVideo << QRgb(0x0000FF) << QtVideo::Rotation::Clockwise90;
-    QTest::addRow("180 deg clockwise") << m_colorMatrix180degClockwiseVideo << QRgb(0xFFFF00) << QtVideo::Rotation::Clockwise180;
-    QTest::addRow("270 deg clockwise") << m_colorMatrix270degClockwiseVideo << QRgb(0x00FF00) << QtVideo::Rotation::Clockwise270;
+    QTest::addColumn<QSize>("videoSize");
+
+    // clang-format off
+    QTest::addRow("without rotation") << m_colorMatrixVideo
+                                      << QRgb(0xff0000)
+                                      << QtVideo::Rotation::None
+                                      << QSize(960, 540);
+
+    QTest::addRow("90 deg clockwise") << m_colorMatrix90degClockwiseVideo
+                                      << QRgb(0x0000FF)
+                                      << QtVideo::Rotation::Clockwise90
+                                      << QSize(540, 960);
+
+    QTest::addRow("180 deg clockwise") << m_colorMatrix180degClockwiseVideo
+                                       << QRgb(0xFFFF00)
+                                       << QtVideo::Rotation::Clockwise180
+                                       << QSize(960, 540);
+
+    QTest::addRow("270 deg clockwise") << m_colorMatrix270degClockwiseVideo
+                                       << QRgb(0x00FF00)
+                                       << QtVideo::Rotation::Clockwise270
+                                       << QSize(540, 960);
+    // clang-format on
 }
 
 void tst_QMediaPlayerBackend::play_playsRotatedVideoOutput_whenVideoFileHasOrientationMetadata()
@@ -2861,9 +2880,10 @@ void tst_QMediaPlayerBackend::play_playsRotatedVideoOutput_whenVideoFileHasOrien
     // viewed with a 90, 180 and 270 degree clockwise rotation respectively.
 
     // Fetch path and expected color of upper left area of each file
-    QFETCH(MaybeUrl, fileURL);
-    QFETCH(QRgb, expectedColor);
-    QFETCH(QtVideo::Rotation, expectedRotationAngle);
+    QFETCH(const MaybeUrl, fileURL);
+    QFETCH(const QRgb, expectedColor);
+    QFETCH(const QtVideo::Rotation, expectedRotationAngle);
+    QFETCH(const QSize, videoSize);
 
     CHECK_SELECTED_URL(fileURL);
 
@@ -2871,6 +2891,9 @@ void tst_QMediaPlayerBackend::play_playsRotatedVideoOutput_whenVideoFileHasOrien
     m_fixture->player.setSource(*fileURL);
     QTRY_COMPARE(m_fixture->player.mediaStatus(), QMediaPlayer::LoadedMedia);
 
+    // Compare videoSize of the output video sink with the expected value before starting playing
+    QCOMPARE(m_fixture->surface.videoSize(), videoSize);
+
     // Compare orientation metadata of QMediaPlayer with expected value
     const auto metaData = m_fixture->player.metaData();
     const auto playerOrientation = metaData.value(QMediaMetaData::Orientation).value<QtVideo::Rotation>();
@@ -2892,6 +2915,9 @@ void tst_QMediaPlayerBackend::play_playsRotatedVideoOutput_whenVideoFileHasOrien
     QVERIFY(!image.isNull());
     QRgb upperLeftColor = image.pixel(5, 5);
     QCOMPARE_LT(colorDifference(upperLeftColor, expectedColor), 0.004);
+
+    // Compare videoSize of the output video sink with the expected value after getting a frame
+    QCOMPARE(m_fixture->surface.videoSize(), videoSize);
 }
 
 QTEST_MAIN(tst_QMediaPlayerBackend)
diff --git a/tests/auto/unit/multimedia/qmultimediautils/tst_qmultimediautils.cpp b/tests/auto/unit/multimedia/qmultimediautils/tst_qmultimediautils.cpp
index bff68b92a..9899013cd 100644
--- a/tests/auto/unit/multimedia/qmultimediautils/tst_qmultimediautils.cpp
+++ b/tests/auto/unit/multimedia/qmultimediautils/tst_qmultimediautils.cpp
@@ -17,6 +17,8 @@ private slots:
     void fraction_of_29_97();
     void fraction_of_lower_boundary();
     void fraction_of_upper_boundary();
+
+    void qRotatedFrameSize_returnsSizeAccordinglyToRotation();
 };
 
 void tst_QMultimediaUtils::fraction_of_0()
@@ -71,5 +73,25 @@ void tst_QMultimediaUtils::fraction_of_upper_boundary()
     QVERIFY(double(n) / double(d) > f);
 }
 
+void tst_QMultimediaUtils::qRotatedFrameSize_returnsSizeAccordinglyToRotation()
+{
+    QCOMPARE(qRotatedFrameSize({ 10, 22 }, 0), QSize(10, 22));
+    QCOMPARE(qRotatedFrameSize({ 10, 23 }, -180), QSize(10, 23));
+    QCOMPARE(qRotatedFrameSize({ 10, 24 }, 180), QSize(10, 24));
+    QCOMPARE(qRotatedFrameSize({ 10, 25 }, 360), QSize(10, 25));
+    QCOMPARE(qRotatedFrameSize({ 11, 26 }, 540), QSize(11, 26));
+
+    QCOMPARE(qRotatedFrameSize({ 10, 22 }, -90), QSize(22, 10));
+    QCOMPARE(qRotatedFrameSize({ 10, 23 }, 90), QSize(23, 10));
+    QCOMPARE(qRotatedFrameSize({ 10, 24 }, 270), QSize(24, 10));
+    QCOMPARE(qRotatedFrameSize({ 10, 25 }, 450), QSize(25, 10));
+
+    QCOMPARE(qRotatedFrameSize({ 10, 22 }, QtVideo::Rotation::None), QSize(10, 22));
+    QCOMPARE(qRotatedFrameSize({ 10, 22 }, QtVideo::Rotation::Clockwise180), QSize(10, 22));
+
+    QCOMPARE(qRotatedFrameSize({ 11, 22 }, QtVideo::Rotation::Clockwise90), QSize(22, 11));
+    QCOMPARE(qRotatedFrameSize({ 11, 22 }, QtVideo::Rotation::Clockwise270), QSize(22, 11));
+}
+
 QTEST_MAIN(tst_QMultimediaUtils)
 #include "tst_qmultimediautils.moc"