Android: Enable FFmpeg playback - HW rendering

This patch enables video playback on Android using Hardware rendering
with FFmpeg backend.

Task-number: QTBUG-102233
Change-Id: Ib9af89ef78e53846d0742a828a3ec5430a8a9f80
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

7 files changed, 250 insertions, 1 deletions

diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
index 402a3b0aa..9185d61ce 100644
--- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt
+++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
@@ -86,8 +86,13 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_linux_v4l
 
 if (ANDROID)
     qt_internal_extend_target(QFFmpegMediaPlugin
+        SOURCES
+            qffmpeghwaccel_mediacodec.cpp qffmpeghwaccel_mediacodec_p.h
+            ../android/wrappers/jni/androidsurfacetexture_p.h
+            ../android/wrappers/jni/androidsurfacetexture.cpp
         INCLUDE_DIRECTORIES
             ${FFMPEG_DIR}/include
+            ../android/wrappers/jni/
     )
 
     set_property(TARGET QFFmpegMediaPlugin APPEND PROPERTY QT_ANDROID_LIB_DEPENDENCIES
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp
index 31b334fbc..3aaf1f781 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp
@@ -90,7 +90,9 @@ Codec::Codec(AVFormatContext *format, int streamIndex)
     Q_ASSERT(streamIndex >= 0 && streamIndex < (int)format->nb_streams);
 
     AVStream *stream = format->streams[streamIndex];
-    const AVCodec *decoder = avcodec_find_decoder(stream->codecpar->codec_id);
+    const AVCodec *decoder =
+            QFFmpeg::HWAccel::hardwareDecoderForCodecId(stream->codecpar->codec_id);
+
     if (!decoder) {
         qCDebug(qLcDecoder) << "Failed to find a valid FFmpeg decoder";
         return;
@@ -705,6 +707,18 @@ void VideoRenderer::loop()
     if (sink) {
         qint64 startTime = frame.pts();
 //        qDebug() << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi();
+
+        // in practice this only happens with mediacodec
+        if (!frame.codec()->hwAccel().isNull() && !frame.avFrame()->hw_frames_ctx) {
+            HWAccel hwaccel = frame.codec()->hwAccel();
+            AVFrame *avframe = frame.avFrame();
+            if (!hwaccel.hwFramesContext())
+                hwaccel.createFramesContext(AVPixelFormat(avframe->format),
+                                            { avframe->width, avframe->height });
+
+            avframe->hw_frames_ctx = av_buffer_ref(hwaccel.hwFramesContextAsBuffer());
+        }
+
         QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(frame.takeAVFrame());
         QVideoFrameFormat format(buffer->size(), buffer->pixelFormat());
         format.setColorSpace(buffer->colorSpace());
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
index e28a89ad8..318db4186 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
@@ -47,6 +47,9 @@
 #if QT_CONFIG(wmf)
 #include "qffmpeghwaccel_d3d11_p.h"
 #endif
+#ifdef Q_OS_ANDROID
+#    include "qffmpeghwaccel_mediacodec_p.h"
+#endif
 #include "qffmpeg_p.h"
 #include "qffmpegvideobuffer_p.h"
 
@@ -145,6 +148,10 @@ AVPixelFormat getFormat(AVCodecContext *s, const AVPixelFormat *fmt)
                     if (fmt[n] == AV_PIX_FMT_D3D11)
                         QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(s);
 #endif
+#ifdef Q_OS_ANDROID
+                    if (fmt[n] == AV_PIX_FMT_MEDIACODEC)
+                        QFFmpeg::MediaCodecTextureConverter::setupDecoderSurface(s);
+#endif
                     return fmt[n];
                 }
             }
@@ -230,11 +237,45 @@ AVPixelFormat HWAccel::hwFormat() const
         return AV_PIX_FMT_VIDEOTOOLBOX;
     case AV_HWDEVICE_TYPE_VAAPI:
         return AV_PIX_FMT_VAAPI;
+    case AV_HWDEVICE_TYPE_MEDIACODEC:
+        return AV_PIX_FMT_MEDIACODEC;
     default:
         return AV_PIX_FMT_NONE;
     }
 }
 
+const AVCodec *HWAccel::hardwareDecoderForCodecId(AVCodecID id)
+{
+    const auto getDecoder = [](AVCodecID id) {
+        switch (id) {
+#ifdef Q_OS_ANDROID
+        case AV_CODEC_ID_H264:
+            return avcodec_find_decoder_by_name("h264_mediacodec");
+        case AV_CODEC_ID_HEVC:
+            return avcodec_find_decoder_by_name("hevc_mediacodec");
+        case AV_CODEC_ID_MPEG2VIDEO:
+            return avcodec_find_decoder_by_name("mpeg2_mediacodec");
+        case AV_CODEC_ID_MPEG4:
+            return avcodec_find_decoder_by_name("mpeg4_mediacodec");
+        case AV_CODEC_ID_VP8:
+            return avcodec_find_decoder_by_name("vp8_mediacodec");
+        case AV_CODEC_ID_VP9:
+            return avcodec_find_decoder_by_name("vp9_mediacodec");
+#endif
+        default:
+            return avcodec_find_decoder(id);
+        }
+    };
+
+    const auto *codec = getDecoder(id);
+
+    // only if ffmpeg build was not setup properly
+    if (Q_UNLIKELY(!codec))
+        codec = avcodec_find_decoder(id);
+
+    return codec;
+}
+
 const AVCodec *HWAccel::hardwareEncoderForCodecId(AVCodecID id) const
 {
     const char *codec = nullptr;
@@ -374,6 +415,11 @@ void TextureConverter::updateBackend(AVPixelFormat fmt)
         d->backend = new D3D11TextureConverter(d->rhi);
         break;
 #endif
+#ifdef Q_OS_ANDROID
+    case AV_PIX_FMT_MEDIACODEC:
+        d->backend = new MediaCodecTextureConverter(d->rhi);
+        break;
+#endif
     default:
         break;
     }
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp
new file mode 100644
index 000000000..e8872a1eb
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp
@@ -0,0 +1,106 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qffmpeghwaccel_mediacodec_p.h"
+
+#include <androidsurfacetexture_p.h>
+#include <QtGui/private/qrhi_p.h>
+
+extern "C" {
+#include <libavcodec/mediacodec.h>
+}
+
+#if !defined(Q_OS_ANDROID)
+#    error "Configuration error"
+#endif
+
+namespace QFFmpeg {
+
+Q_GLOBAL_STATIC(AndroidSurfaceTexture, androidSurfaceTexture, 0);
+
+class MediaCodecTextureSet : public TextureSet
+{
+public:
+    MediaCodecTextureSet(qint64 textureHandle) : handle(textureHandle) { }
+
+    qint64 textureHandle(int plane) override { return (plane == 0) ? handle : 0; }
+
+private:
+    qint64 handle;
+};
+
+void MediaCodecTextureConverter::setupDecoderSurface(AVCodecContext *avCodecContext)
+{
+    AVMediaCodecContext *mediacodecContext = av_mediacodec_alloc_context();
+    av_mediacodec_default_init(avCodecContext, mediacodecContext, androidSurfaceTexture->surface());
+}
+
+TextureSet *MediaCodecTextureConverter::getTextures(AVFrame *frame)
+{
+    if (!androidSurfaceTexture->isValid())
+        return {};
+
+    if (!externalTexture) {
+        androidSurfaceTexture->detachFromGLContext();
+        externalTexture = std::unique_ptr<QRhiTexture>(
+                rhi->newTexture(QRhiTexture::Format::RGBA8, { frame->width, frame->height }, 1,
+                                QRhiTexture::ExternalOES));
+
+        if (!externalTexture->create()) {
+            qWarning() << "Failed to create the external texture!";
+            return {};
+        }
+
+        quint64 textureHandle = externalTexture->nativeTexture().object;
+        androidSurfaceTexture->attachToGLContext(textureHandle);
+    }
+
+    // release a MediaCodec buffer and render it to the surface
+    AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)frame->data[3];
+    int result = av_mediacodec_release_buffer(buffer, 1);
+    if (result < 0) {
+        qWarning() << "Failed to render buffer to surface.";
+        return {};
+    }
+
+    androidSurfaceTexture->updateTexImage();
+
+    return new MediaCodecTextureSet(externalTexture->nativeTexture().object);
+}
+}
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h
new file mode 100644
index 000000000..df6dd091f
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h
@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QFFMPEGHWACCEL_MEDIACODEC_P_H
+#define QFFMPEGHWACCEL_MEDIACODEC_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "qffmpeghwaccel_p.h"
+#include <memory>
+
+namespace QFFmpeg {
+struct Frame;
+
+class MediaCodecTextureConverter : public TextureConverterBackend
+{
+public:
+    MediaCodecTextureConverter(QRhi *rhi) : TextureConverterBackend(rhi){};
+    TextureSet *getTextures(AVFrame *frame) override;
+
+    static void setupDecoderSurface(AVCodecContext *s);
+private:
+    std::unique_ptr<QRhiTexture> externalTexture;
+};
+}
+#endif // QFFMPEGHWACCEL_MEDIACODEC_P_H
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
index c25faecfd..2d6acf1f3 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
@@ -142,6 +142,8 @@ public:
     const AVCodec *hardwareEncoderForCodecId(AVCodecID id) const;
     static HWAccel findHardwareAccelForCodecID(AVCodecID id);
 
+    static const AVCodec *hardwareDecoderForCodecId(AVCodecID id);
+
     void createFramesContext(AVPixelFormat swFormat, const QSize &size);
     AVBufferRef *hwFramesContextAsBuffer() const { return d ? d->hwFramesContext : nullptr; }
     AVHWFramesContext *hwFramesContext() const;
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index 144281896..754c84743 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -309,6 +309,8 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat
         return QVideoFrameFormat::Format_P010;
     case AV_PIX_FMT_P016:
         return QVideoFrameFormat::Format_P016;
+    case AV_PIX_FMT_MEDIACODEC:
+        return QVideoFrameFormat::Format_SamplerExternalOES;
     }
 
     if (needsConversion)
@@ -382,6 +384,9 @@ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat
         return AV_PIX_FMT_P010;
     case QVideoFrameFormat::Format_P016:
         return AV_PIX_FMT_P016;
+
+    case QVideoFrameFormat::Format_SamplerExternalOES:
+        return AV_PIX_FMT_MEDIACODEC;
     }
 }