Modify rule for encoding target format to handle Android corner case

Recently the rule was made stricter, but it appears to be failed on
Android, as mediacodec doesn't accept AV_PIX_FMT_MEDIACODEC.
The matching ticket has ben created.

Task-number: QTBUG-116836
Change-Id: Ia9fa19ead9e236e225750dd4d4bffe45289d8fb3
Reviewed-by: Bartlomiej Moskal <bartlomiej.moskal@qt.io>
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
(cherry picked from commit ff3857e62115f7283c00c84b6f43827e53ac879f)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

2 files changed, 27 insertions, 12 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoencoderutils.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideoencoderutils.cpp
index 2f53412d9..f22b4865d 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideoencoderutils.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideoencoderutils.cpp
@@ -64,6 +64,17 @@ static auto targetSwFormatScoreCalculator(AVPixelFormat sourceFormat)
     return [=](AVPixelFormat fmt) { return calculateTargetSwFormatScore(sourceSwFormatDesc, fmt); };
 }
 
+static bool isHwFormatAcceptedByCodec(AVPixelFormat pixFormat)
+{
+    switch (pixFormat) {
+    case AV_PIX_FMT_MEDIACODEC:
+        // Mediacodec doesn't accept AV_PIX_FMT_MEDIACODEC (QTBUG-116836)
+        return false;
+    default:
+        return true;
+    }
+}
+
 AVPixelFormat findTargetSWFormat(AVPixelFormat sourceSWFormat, const AVCodec *codec,
                                  const HWAccel &accel)
 {
@@ -84,19 +95,22 @@ AVPixelFormat findTargetSWFormat(AVPixelFormat sourceSWFormat, const AVCodec *co
 AVPixelFormat findTargetFormat(AVPixelFormat sourceFormat, AVPixelFormat sourceSWFormat,
                                const AVCodec *codec, const HWAccel *accel)
 {
+    Q_UNUSED(sourceFormat);
+
     if (accel) {
         const auto hwFormat = accel->hwFormat();
 
+        // TODO: handle codec->capabilities & AV_CODEC_CAP_HARDWARE here
+        if (!isHwFormatAcceptedByCodec(hwFormat))
+            return findTargetSWFormat(sourceSWFormat, codec, *accel);
+
         const auto constraints = accel->constraints();
         if (constraints && hasAVFormat(constraints->valid_hw_formats, hwFormat))
             return hwFormat;
 
-        // Some codecs, e.g. mediacodec, don't expose constraints, let's find the format in
-        // codec->pix_fmts
+        // Some codecs, don't expose constraints, let's find the format in codec->pix_fmts
         if (hasAVFormat(codec->pix_fmts, hwFormat))
             return hwFormat;
-
-        return AV_PIX_FMT_NONE;
     }
 
     if (!codec->pix_fmts) {
@@ -105,8 +119,8 @@ AVPixelFormat findTargetFormat(AVPixelFormat sourceFormat, AVPixelFormat sourceS
         return sourceSWFormat;
     }
 
-    auto scoreCalculator = targetSwFormatScoreCalculator(sourceFormat);
-    return findBestAVFormat(codec->pix_fmts, scoreCalculator).first;
+    auto swScoreCalculator = targetSwFormatScoreCalculator(sourceSWFormat);
+    return findBestAVFormat(codec->pix_fmts, swScoreCalculator).first;
 }
 
 std::pair<const AVCodec *, std::unique_ptr<HWAccel>> findHwEncoder(AVCodecID codecID,
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp
index 8e3b74929..7d6afb522 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp
@@ -29,7 +29,9 @@ VideoFrameEncoder::create(const QMediaEncoderSettings &encoderSettings, const QS
     result->m_settings = encoderSettings;
     result->m_sourceSize = sourceSize;
     result->m_sourceFormat = sourceFormat;
-    result->m_sourceSWFormat = sourceSWFormat;
+
+    // Temporary: check isSwPixelFormat because of android issue (QTBUG-116836)
+    result->m_sourceSWFormat = isSwPixelFormat(sourceFormat) ? sourceFormat : sourceSWFormat;
 
     if (!result->m_settings.videoResolution().isValid())
         result->m_settings.setVideoResolution(sourceSize);
@@ -112,9 +114,9 @@ bool VideoFrameEncoder::initTargetFormats()
         return false;
     }
 
-    Q_ASSERT(isHwPixelFormat(m_targetFormat) == !!m_accel);
+    if (isHwPixelFormat(m_targetFormat)) {
+        Q_ASSERT(m_accel);
 
-    if (m_accel) {
         m_targetSWFormat = findTargetSWFormat(m_sourceSWFormat, m_codec, *m_accel);
 
         if (m_targetSWFormat == AV_PIX_FMT_NONE) {
@@ -179,9 +181,8 @@ bool QFFmpeg::VideoFrameEncoder::initCodecContext(AVFormatContext *formatContext
         Q_ASSERT(deviceContext);
         m_codecContext->hw_device_ctx = av_buffer_ref(deviceContext);
 
-        auto framesContext = m_accel->hwFramesContextAsBuffer();
-        Q_ASSERT(deviceContext);
-        m_codecContext->hw_frames_ctx = av_buffer_ref(framesContext);
+        if (auto framesContext = m_accel->hwFramesContextAsBuffer())
+            m_codecContext->hw_frames_ctx = av_buffer_ref(framesContext);
     }
 
     return true;