Handle non-standard pixel aspect ratios by the ffmpeg backend

Videos may have some specific pixel aspect ratio which means
that video frames should be scaled on drawing.

The patch handles such videos an makes them draw properly.

Task-number: QTBUG-108754
Pick-to: 6.5
Change-Id: Iddfa37b55bccde35d97e64ac58b4f639ea51451b
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 2d5a8d28841fba0b0c4d8ed67dcf91f3bfafe486)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

11 files changed, 152 insertions, 22 deletions

diff --git a/src/multimedia/qmultimediautils.cpp b/src/multimedia/qmultimediautils.cpp
index 5d497f74d..26b71cad6 100644
--- a/src/multimedia/qmultimediautils.cpp
+++ b/src/multimedia/qmultimediautils.cpp
@@ -37,4 +37,15 @@ Fraction qRealToFraction(qreal value)
         return { n1 + integral * d1, d1 };
 }
 
+QSize qCalculateFrameSize(QSize resolution, Fraction par)
+{
+    if (par.numerator == par.denominator || par.numerator < 1 || par.denominator < 1)
+        return resolution;
+
+    if (par.numerator > par.denominator)
+        return { resolution.width() * par.numerator / par.denominator, resolution.height() };
+
+    return { resolution.width(), resolution.height() * par.denominator / par.numerator };
+}
+
 QT_END_NAMESPACE
diff --git a/src/multimedia/qmultimediautils_p.h b/src/multimedia/qmultimediautils_p.h
index 0a49afa4b..74dde7893 100644
--- a/src/multimedia/qmultimediautils_p.h
+++ b/src/multimedia/qmultimediautils_p.h
@@ -18,6 +18,7 @@
 #include <QtMultimedia/qtmultimediaglobal.h>
 #include <QtCore/private/qglobal_p.h>
 #include <qstring.h>
+#include <qsize.h>
 #include <utility>
 #include <optional>
 
@@ -94,6 +95,8 @@ struct Fraction {
 
 Q_MULTIMEDIA_EXPORT Fraction qRealToFraction(qreal value);
 
+Q_MULTIMEDIA_EXPORT QSize qCalculateFrameSize(QSize resolution, Fraction pixelAspectRatio);
+
 QT_END_NAMESPACE
 
 #endif // QMULTIMEDIAUTILS_P_H
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp
index 48b8c6898..533c49b7a 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp
@@ -10,10 +10,12 @@ static Q_LOGGING_CATEGORY(qLcPlaybackEngineCodec, "qt.multimedia.playbackengine.
 
 namespace QFFmpeg {
 
-Codec::Data::Data(AVCodecContextUPtr context, AVStream *stream,
+Codec::Data::Data(AVCodecContextUPtr context, AVStream *stream, AVFormatContext *formatContext,
                   std::unique_ptr<QFFmpeg::HWAccel> hwAccel)
     : context(std::move(context)), stream(stream), hwAccel(std::move(hwAccel))
 {
+    if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
+        pixelAspectRatio = av_guess_sample_aspect_ratio(formatContext, stream, nullptr);
 }
 
 Codec::Data::~Data()
@@ -23,7 +25,7 @@ Codec::Data::~Data()
     avcodec_close(context.get());
 }
 
-QMaybe<Codec> Codec::create(AVStream *stream)
+QMaybe<Codec> Codec::create(AVStream *stream, AVFormatContext *formatContext)
 {
     if (!stream)
         return { "Invalid stream" };
@@ -71,7 +73,14 @@ QMaybe<Codec> Codec::create(AVStream *stream)
     if (ret < 0)
         return QString("Failed to open FFmpeg codec context " + err2str(ret));
 
-    return Codec(new Data(std::move(context), stream, std::move(hwAccel)));
+    return Codec(new Data(std::move(context), stream, formatContext, std::move(hwAccel)));
+}
+
+AVRational Codec::pixelAspectRatio(AVFrame *frame) const
+{
+    // does the same as av_guess_sample_aspect_ratio, but more efficient
+    return d->pixelAspectRatio.num && d->pixelAspectRatio.den ? d->pixelAspectRatio
+                                                              : frame->sample_aspect_ratio;
 }
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h
index 4437c4df1..5510e0e84 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h
@@ -29,17 +29,20 @@ class Codec
 {
     struct Data
     {
-        Data(AVCodecContextUPtr context, AVStream *stream,
+        Data(AVCodecContextUPtr context, AVStream *stream, AVFormatContext *formatContext,
              std::unique_ptr<QFFmpeg::HWAccel> hwAccel);
         ~Data();
         QAtomicInt ref;
         AVCodecContextUPtr context;
         AVStream *stream = nullptr;
+        AVRational pixelAspectRatio = { 0, 1 };
         std::unique_ptr<QFFmpeg::HWAccel> hwAccel;
     };
 
 public:
-    static QMaybe<Codec> create(AVStream *);
+    static QMaybe<Codec> create(AVStream *stream, AVFormatContext *formatContext);
+
+    AVRational pixelAspectRatio(AVFrame *frame) const;
 
     AVCodecContext *context() const { return d->context.get(); }
     AVStream *stream() const { return d->stream; }
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp
index 2869d2c34..4f1555034 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp
@@ -34,14 +34,17 @@ VideoRenderer::RenderingResult VideoRenderer::renderInternal(Frame frame)
 
     //        qCDebug(qLcVideoRenderer) << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi();
 
+    const auto codec = frame.codec();
+    Q_ASSERT(codec);
+
 #ifdef Q_OS_ANDROID
     // QTBUG-108446
     // In general case, just creation of frames context is not correct since
     //   frames may require additional specific data for hw contexts, so
     //   just setting of hw_frames_ctx is not enough.
     // TODO: investigate the case in order to remove or fix the code.
-    if (frame.codec()->hwAccel() && !frame.avFrame()->hw_frames_ctx) {
-        HWAccel *hwaccel = frame.codec()->hwAccel();
+    if (codec->hwAccel() && !frame.avFrame()->hw_frames_ctx) {
+        HWAccel *hwaccel = codec->hwAccel();
         AVFrame *avframe = frame.avFrame();
         if (!hwaccel->hwFramesContext())
             hwaccel->createFramesContext(AVPixelFormat(avframe->format),
@@ -52,7 +55,8 @@ VideoRenderer::RenderingResult VideoRenderer::renderInternal(Frame frame)
     }
 #endif
 
-    auto buffer = std::make_unique<QFFmpegVideoBuffer>(frame.takeAVFrame());
+    const auto pixelAspectRatio = codec->pixelAspectRatio(frame.avFrame());
+    auto buffer = std::make_unique<QFFmpegVideoBuffer>(frame.takeAVFrame(), pixelAspectRatio);
     QVideoFrameFormat format(buffer->size(), buffer->pixelFormat());
     format.setColorSpace(buffer->colorSpace());
     format.setColorTransfer(buffer->colorTransfer());
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp
index 964125bff..5ec98fc14 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp
@@ -377,7 +377,8 @@ std::optional<Codec> PlaybackEngine::codecForTrack(QPlatformMediaPlayer::TrackTy
     if (!result) {
         qCDebug(qLcPlaybackEngine)
                 << "Create codec for stream:" << streamIndex << "trackType:" << trackType;
-        auto maybeCodec = Codec::create(m_media.avContext()->streams[streamIndex]);
+        auto maybeCodec =
+                Codec::create(m_media.avContext()->streams[streamIndex], m_media.avContext());
 
         if (!maybeCodec) {
             emit errorOccured(QMediaPlayer::FormatError,
@@ -606,8 +607,18 @@ void PlaybackEngine::updateVideoSinkSize(QVideoSink *prevSink)
 
     if (prevSink && prevSink->platformVideoSink())
         platformVideoSink->setNativeSize(prevSink->platformVideoSink()->nativeSize());
-    else if (auto size = metaData().value(QMediaMetaData::Resolution); size.isValid())
-        platformVideoSink->setNativeSize(size.value<QSize>());
+    else {
+        const auto streamIndex = m_media.currentStreamIndex(QPlatformMediaPlayer::VideoStream);
+        if (streamIndex >= 0) {
+            const auto context = m_media.avContext();
+            const auto stream = context->streams[streamIndex];
+            const auto pixelAspectRatio = av_guess_sample_aspect_ratio(context, stream, nullptr);
+            // auto size = metaData().value(QMediaMetaData::Resolution)
+            platformVideoSink->setNativeSize(
+                    qCalculateFrameSize({ stream->codecpar->width, stream->codecpar->height },
+                                        { pixelAspectRatio.num, pixelAspectRatio.den }));
+        }
+    }
 }
 
 qint64 PlaybackEngine::boundPosition(qint64 position) const
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index 7b72a3cfe..e1bd93537 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -3,6 +3,7 @@
 
 #include "qffmpegvideobuffer_p.h"
 #include "private/qvideotexturehelper_p.h"
+#include "private/qmultimediautils_p.h"
 #include "qffmpeghwaccel_p.h"
 
 extern "C" {
@@ -22,8 +23,11 @@ static bool isFrameFlipped(const AVFrame& frame) {
 
 QT_BEGIN_NAMESPACE
 
-QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrameUPtr frame)
-    : QAbstractVideoBuffer(QVideoFrame::NoHandle), frame(frame.get())
+QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrameUPtr frame, AVRational pixelAspectRatio)
+    : QAbstractVideoBuffer(QVideoFrame::NoHandle),
+      frame(frame.get()),
+      m_size(qCalculateFrameSize({ frame->width, frame->height },
+                                 { pixelAspectRatio.num, pixelAspectRatio.den }))
 {
     if (frame->hw_frames_ctx) {
         hwFrame = std::move(frame);
@@ -45,16 +49,18 @@ void QFFmpegVideoBuffer::convertSWFrame()
 
     const auto actualAVPixelFormat = AVPixelFormat(swFrame->format);
     const auto targetAVPixelFormat = toAVPixelFormat(m_pixelFormat);
-    if (actualAVPixelFormat != targetAVPixelFormat || isFrameFlipped(*swFrame)) {
+
+    if (actualAVPixelFormat != targetAVPixelFormat || isFrameFlipped(*swFrame)
+        || m_size != QSize(swFrame->width, swFrame->height)) {
         Q_ASSERT(toQtPixelFormat(targetAVPixelFormat) == m_pixelFormat);
         // convert the format into something we can handle
         SwsContext *c = sws_getContext(swFrame->width, swFrame->height, actualAVPixelFormat,
-                                       swFrame->width, swFrame->height, targetAVPixelFormat,
+                                       m_size.width(), m_size.height(), targetAVPixelFormat,
                                        SWS_BICUBIC, nullptr, nullptr, nullptr);
 
         auto newFrame = QFFmpeg::makeAVFrame();
-        newFrame->width = swFrame->width;
-        newFrame->height = swFrame->height;
+        newFrame->width = m_size.width();
+        newFrame->height = m_size.height();
         newFrame->format = targetAVPixelFormat;
         av_frame_get_buffer(newFrame.get(), 0);
 
@@ -230,7 +236,7 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::pixelFormat() const
 
 QSize QFFmpegVideoBuffer::size() const
 {
-    return QSize(frame->width, frame->height);
+    return m_size;
 }
 
 QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat avPixelFormat, bool *needsConversion)
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h
index 06ba8ca54..b7249b8d5 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h
@@ -30,7 +30,7 @@ class QFFmpegVideoBuffer : public QAbstractVideoBuffer
 public:
     using AVFrameUPtr = QFFmpeg::AVFrameUPtr;
 
-    QFFmpegVideoBuffer(AVFrameUPtr frame);
+    QFFmpegVideoBuffer(AVFrameUPtr frame, AVRational pixelAspectRatio = { 1, 1 });
     ~QFFmpegVideoBuffer() override;
 
     QVideoFrame::MapMode mapMode() const override;
@@ -63,6 +63,7 @@ private:
     AVFrame *frame = nullptr;
     AVFrameUPtr hwFrame;
     AVFrameUPtr swFrame;
+    QSize m_size;
     QFFmpeg::TextureConverter textureConverter;
     QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped;
     std::unique_ptr<QFFmpeg::TextureSet> textures;
diff --git a/tests/auto/integration/qmediaplayerbackend/testdata/par_2_3.mp4 b/tests/auto/integration/qmediaplayerbackend/testdata/par_2_3.mp4
new file mode 100644
index 000000000..6392b8f1f
--- /dev/null
+++ b/tests/auto/integration/qmediaplayerbackend/testdata/par_2_3.mp4
diff --git a/tests/auto/integration/qmediaplayerbackend/testdata/par_3_2.mp4 b/tests/auto/integration/qmediaplayerbackend/testdata/par_3_2.mp4
new file mode 100644
index 000000000..a277d6a90
--- /dev/null
+++ b/tests/auto/integration/qmediaplayerbackend/testdata/par_3_2.mp4
diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
index 892efbe16..91697375e 100644
--- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
+++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
@@ -98,6 +98,9 @@ private slots:
     void setSource_remainsInStoppedState_whenPlayerWasStopped();
     void setSource_entersStoppedState_whenPlayerWasPlaying();
 
+    void setSourceAndPlay_setCorrectVideoSize_whenVideoHasNonStandardPixelAspectRatio_data();
+    void setSourceAndPlay_setCorrectVideoSize_whenVideoHasNonStandardPixelAspectRatio();
+
     void pause_doesNotChangePlayerState_whenInvalidFileLoaded();
     void pause_doesNothing_whenMediaIsNotLoaded();
     void pause_entersPauseState_whenPlayerWasPlaying();
@@ -178,6 +181,8 @@ private:
     QUrl m_localFileWithMetadata;
     QUrl m_localVideoFile3ColorsWithSound;
     QUrl m_oneRedFrameVideo;
+    QUrl m_192x108_PAR_2_3_Video;
+    QUrl m_192x108_PAR_3_2_Video;
 
     const std::array<QRgb, 3> m_video3Colors = { { 0xFF0000, 0x00FF00, 0x0000FF } };
     QString m_vlcCommand;
@@ -295,6 +300,11 @@ void tst_QMediaPlayerBackend::initTestCase()
     m_oneRedFrameVideo =
             MediaFileSelector::selectMediaFile(QStringList() << "qrc:/testdata/one_red_frame.mp4");
 
+    m_192x108_PAR_2_3_Video =
+            MediaFileSelector::selectMediaFile(QStringList() << "qrc:/testdata/par_2_3.mp4");
+    m_192x108_PAR_3_2_Video =
+            MediaFileSelector::selectMediaFile(QStringList() << "qrc:/testdata/par_3_2.mp4");
+
     detectVlcCommand();
 }
 
@@ -696,6 +706,78 @@ void tst_QMediaPlayerBackend::setSource_entersStoppedState_whenPlayerWasPlaying(
     QCOMPARE(m_fixture->player.position(), 0);
 }
 
+void tst_QMediaPlayerBackend::
+        setSourceAndPlay_setCorrectVideoSize_whenVideoHasNonStandardPixelAspectRatio_data()
+{
+    QTest::addColumn<QUrl>("url");
+    QTest::addColumn<QSize>("expectedVideoSize");
+
+    QTest::addRow("Horizontal expanding (par=3/2)")
+            << m_192x108_PAR_3_2_Video << QSize(192 * 3 / 2, 108);
+    QTest::addRow("Vertical expanding (par=2/3)")
+            << m_192x108_PAR_2_3_Video << QSize(192, 108 * 3 / 2);
+}
+
+void tst_QMediaPlayerBackend::
+        setSourceAndPlay_setCorrectVideoSize_whenVideoHasNonStandardPixelAspectRatio()
+{
+    QFETCH(QUrl, url);
+    QFETCH(QSize, expectedVideoSize);
+
+    m_fixture->player.setSource(url);
+    QTRY_COMPARE(m_fixture->player.mediaStatus(), QMediaPlayer::LoadedMedia);
+    QCOMPARE(m_fixture->player.metaData().value(QMediaMetaData::Resolution), QSize(192, 108));
+
+    QCOMPARE(m_fixture->surface.videoSize(), expectedVideoSize);
+
+    m_fixture->player.play();
+
+    auto frame = m_fixture->surface.waitForFrame();
+    QVERIFY(frame.isValid());
+    QCOMPARE(frame.size(), expectedVideoSize);
+    QCOMPARE(frame.surfaceFormat().frameSize(), expectedVideoSize);
+    QCOMPARE(frame.surfaceFormat().viewport(), QRect(QPoint(), expectedVideoSize));
+
+    auto image = frame.toImage();
+    QCOMPARE(frame.size(), expectedVideoSize);
+
+    // clang-format off
+
+    // Video schema:
+    //
+    //           192
+    // /---------------------\
+    // |   White  |          |
+    // |          |          |
+    // |----------/          | 108
+    // |              Red    |
+    // |                     |
+    // \---------------------/
+
+    // clang-format on
+
+    // check the proper scaling
+    const std::vector<QRgb> colors = { 0xFFFFFF, 0xFF0000, 0xFF00, 0xFF, 0x0 };
+
+    const auto pixelsOffset = 4;
+    const auto halfSize = expectedVideoSize / 2;
+
+    QCOMPARE(findSimilarColorIndex(colors, image.pixel(0, 0)), 0);
+    QCOMPARE(findSimilarColorIndex(colors, image.pixel(halfSize.width() - pixelsOffset, 0)), 0);
+    QCOMPARE(findSimilarColorIndex(colors, image.pixel(0, halfSize.height() - pixelsOffset)), 0);
+    QCOMPARE(findSimilarColorIndex(colors,
+                                   image.pixel(halfSize.width() - pixelsOffset,
+                                               halfSize.height() - pixelsOffset)),
+             0);
+
+    QCOMPARE(findSimilarColorIndex(colors, image.pixel(halfSize.width() + pixelsOffset, 0)), 1);
+    QCOMPARE(findSimilarColorIndex(colors, image.pixel(0, halfSize.height() + pixelsOffset)), 1);
+    QCOMPARE(findSimilarColorIndex(colors,
+                                   image.pixel(halfSize.width() + pixelsOffset,
+                                               halfSize.height() + pixelsOffset)),
+             1);
+}
+
 void tst_QMediaPlayerBackend::pause_doesNotChangePlayerState_whenInvalidFileLoaded()
 {
     m_fixture->player.setSource({ "Some not existing media" });
@@ -929,7 +1011,7 @@ void tst_QMediaPlayerBackend::
 
         auto frame1 = surface.waitForFrame();
         QVERIFY(frame1.isValid());
-        QCOMPARE(frame1.size(), QSize(160, 120));
+        QCOMPARE(frame1.size(), QSize(213, 120));
 
         QCOMPARE_GT(frame1.startTime(), pos * 1000);
 
@@ -1404,7 +1486,7 @@ void tst_QMediaPlayerBackend::seekPauseSeek()
         QVideoFrame frame = surface.m_frameList.back();
         const qint64 elapsed = (frame.startTime() / 1000) - position; // frame.startTime() is microsecond, position is milliseconds.
         QVERIFY2(qAbs(elapsed) < (qint64)500, QByteArray::number(elapsed).constData());
-        QCOMPARE(frame.width(), 160);
+        QCOMPARE(frame.width(), 213);
         QCOMPARE(frame.height(), 120);
 
         // create QImage for QVideoFrame to verify RGB pixel colors
@@ -1427,7 +1509,7 @@ void tst_QMediaPlayerBackend::seekPauseSeek()
         QVideoFrame frame = surface.m_frameList.back();
         const qint64 elapsed = (frame.startTime() / 1000) - position;
         QVERIFY2(qAbs(elapsed) < (qint64)500, QByteArray::number(elapsed).constData());
-        QCOMPARE(frame.width(), 160);
+        QCOMPARE(frame.width(), 213);
         QCOMPARE(frame.height(), 120);
 
         QImage image = frame.toImage();