Fix ffmpeg camera crash when videotoolbox doesn't support format

The change just fixes the crash: hw accel is not valid if
videotoolbox doesn't support core format (e.g. 'yuvs', 'yuvf').
Ffmpeg camera still doesn't work with these core formats,
it's to be fixed under the bug QTBUG-109009 (most likely, on ffmpeg
library side).

The issue is reproduced with pretty old cameras, e.g. on
macbook pro 2015.

Fixes: QTBUG-109009
Pick-to: 6.4
Change-Id: Ib36a7e6b3b8c71fa1a3a2717c3509ca30d3b5caa
Reviewed-by: Doris Verria <doris.verria@qt.io>

3 files changed, 30 insertions, 14 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera.mm b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
index d6ec94a0e..9cdb84dc7 100644
--- a/src/plugins/multimedia/ffmpeg/qavfcamera.mm
+++ b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
@@ -361,17 +361,29 @@ void QAVFCamera::updateCameraFormat()
     if (!captureDevice)
         return;
 
-    uint avPixelFormat = 0;
+    std::uint32_t cvPixelFormat = 0;
     AVCaptureDeviceFormat *newFormat = qt_convert_to_capture_device_format(captureDevice, m_cameraFormat);
     if (newFormat) {
         qt_set_active_format(captureDevice, newFormat, false);
-        avPixelFormat = setPixelFormat(m_cameraFormat.pixelFormat());
+        cvPixelFormat = setPixelFormat(m_cameraFormat.pixelFormat());
+    }
+
+    const AVPixelFormat avPixelFormat = av_map_videotoolbox_format_to_pixfmt(cvPixelFormat);
+
+    std::unique_ptr<QFFmpeg::HWAccel> hwAccel;
+
+    if (avPixelFormat == AV_PIX_FMT_NONE) {
+        auto formatDescIt =
+                std::make_reverse_iterator(reinterpret_cast<const char *>(&cvPixelFormat));
+        qWarning() << "Videotoolbox desn't support cvPixelFormat:" << cvPixelFormat
+                   << std::string(formatDescIt - 4, formatDescIt)
+                   << " Camera pix format:" << m_cameraFormat.pixelFormat();
+    } else {
+        hwAccel = QFFmpeg::HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
     }
 
-    auto hwAccel = QFFmpeg::HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
     if (hwAccel) {
-        hwAccel->createFramesContext(av_map_videotoolbox_format_to_pixfmt(avPixelFormat),
-                                     m_cameraFormat.resolution());
+        hwAccel->createFramesContext(avPixelFormat, m_cameraFormat.resolution());
         hwPixelFormat = hwAccel->hwFormat();
     } else {
         hwPixelFormat = AV_PIX_FMT_NONE;
@@ -379,35 +391,36 @@ void QAVFCamera::updateCameraFormat()
     [m_sampleBufferDelegate setHWAccel:std::move(hwAccel)];
 }
 
-uint QAVFCamera::setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat)
+std::uint32_t QAVFCamera::setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat)
 {
     // Default to 32BGRA pixel formats on the viewfinder, in case the requested
     // format can't be used (shouldn't happen unless the developers sets a wrong camera
     // format on the camera).
-    unsigned avPixelFormat = kCVPixelFormatType_32BGRA;
-    if (!QAVFHelpers::toCVPixelFormat(pixelFormat, avPixelFormat))
+    std::uint32_t cvPixelFormat = kCVPixelFormatType_32BGRA;
+    if (!QAVFHelpers::toCVPixelFormat(pixelFormat, cvPixelFormat))
         qWarning() << "QCamera::setCameraFormat: couldn't convert requested pixel format, using ARGB32";
 
     bool isSupported = false;
     NSArray *supportedPixelFormats = m_videoDataOutput.availableVideoCVPixelFormatTypes;
     for (NSNumber *currentPixelFormat in supportedPixelFormats)
     {
-        if ([currentPixelFormat unsignedIntValue] == avPixelFormat) {
+        if ([currentPixelFormat unsignedIntValue] == cvPixelFormat) {
             isSupported = true;
             break;
         }
     }
 
     if (isSupported) {
-        NSDictionary* outputSettings = @{
-            (NSString *)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:avPixelFormat],
-            (NSString *)kCVPixelBufferMetalCompatibilityKey: @true
+        NSDictionary *outputSettings = @{
+            (NSString *)
+            kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithUnsignedInt:cvPixelFormat],
+            (NSString *)kCVPixelBufferMetalCompatibilityKey : @true
         };
         m_videoDataOutput.videoSettings = outputSettings;
     } else {
         qWarning() << "QCamera::setCameraFormat: requested pixel format not supported. Did you use a camera format from another camera?";
     }
-    return avPixelFormat;
+    return cvPixelFormat;
 }
 
 void QAVFCamera::syncHandleFrame(const QVideoFrame &frame)
diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera_p.h b/src/plugins/multimedia/ffmpeg/qavfcamera_p.h
index 40a53dc7c..109682410 100644
--- a/src/plugins/multimedia/ffmpeg/qavfcamera_p.h
+++ b/src/plugins/multimedia/ffmpeg/qavfcamera_p.h
@@ -66,7 +66,7 @@ private:
     void updateCameraFormat();
     void updateVideoInput();
     void attachVideoInputDevice();
-    uint setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat);
+    std::uint32_t setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat);
 
     AVCaptureDevice *device() const;
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index f094ad258..fe12eb4eb 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -221,6 +221,9 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat
     switch (avPixelFormat) {
     default:
         break;
+    case AV_PIX_FMT_NONE:
+        Q_ASSERT(!"Invalid avPixelFormat!");
+        return QVideoFrameFormat::Format_Invalid;
     case AV_PIX_FMT_ARGB:
         return QVideoFrameFormat::Format_ARGB8888;
     case AV_PIX_FMT_0RGB: