QAVFCamera: pass the correct resolution6.5

Which can be affected by the connection.videoOrientation (the sizes of
sample buffers can become 'transposed' - width and height swapped).
So when updating camera format (and there we create HWAccel)
- check if the orientation is 'portrait' but the dimensions are landscape
- transpose such QSize.

Change-Id: Ibd2dac5c532abcc60238bb0e9101195fae79bb1d
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
(cherry picked from commit a669d3e0dca82dde7356385054634c9b4fc453bf)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit 61a7eb181bab7f84e43dc1d05bd74ad08acae08f)

2 files changed, 21 insertions, 1 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera.mm b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
index 800825bd9..9801ad4b9 100644
--- a/src/plugins/multimedia/ffmpeg/qavfcamera.mm
+++ b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
@@ -278,7 +278,7 @@ void QAVFCamera::updateCameraFormat()
     }
 
     if (hwAccel) {
-        hwAccel->createFramesContext(avPixelFormat, m_cameraFormat.resolution());
+        hwAccel->createFramesContext(avPixelFormat, adjustedResolution());
         m_hwPixelFormat = hwAccel->hwFormat();
     } else {
         m_hwPixelFormat = AV_PIX_FMT_NONE;
@@ -340,6 +340,25 @@ uint32_t QAVFCamera::setPixelFormat(QVideoFrameFormat::PixelFormat cameraPixelFo
     return [bestFormat unsignedIntValue];
 }
 
+QSize QAVFCamera::adjustedResolution() const
+{
+    // Check, that we have matching dimesnions.
+    QSize resolution = m_cameraFormat.resolution();
+    AVCaptureConnection *connection = [m_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
+    if (!connection.supportsVideoOrientation)
+        return resolution;
+
+    // Either portrait but actually sizes of landscape, or
+    // landscape with dimensions of portrait - not what
+    // sample delegate will report (it depends on videoOrientation set).
+    const bool isPortraitOrientation = connection.videoOrientation == AVCaptureVideoOrientationPortrait;
+    const bool isPortraitResolution = resolution.height() > resolution.width();
+    if (isPortraitOrientation != isPortraitResolution)
+        resolution.transpose();
+
+    return resolution;
+}
+
 void QAVFCamera::syncHandleFrame(const QVideoFrame &frame)
 {
     Q_EMIT newVideoFrame(frame);
diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera_p.h b/src/plugins/multimedia/ffmpeg/qavfcamera_p.h
index 3f3fc97cd..71aa521bc 100644
--- a/src/plugins/multimedia/ffmpeg/qavfcamera_p.h
+++ b/src/plugins/multimedia/ffmpeg/qavfcamera_p.h
@@ -70,6 +70,7 @@ private:
     void updateVideoInput();
     void attachVideoInputDevice();
     uint32_t setPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat, uint32_t inputCvPixFormat);
+    QSize adjustedResolution() const;
 
     AVCaptureDevice *device() const;