From 4b8b5ad1bd74fd107f034d72c6d44faa386f01f5 Mon Sep 17 00:00:00 2001 From: Samuel Mira Date: Mon, 16 May 2022 20:16:22 +0300 Subject: Android: Enable FFmpeg playback - HW rendering This patch enables video playback on Android using Hardware rendering with FFmpeg backend. Task-number: QTBUG-102233 Change-Id: Ib9af89ef78e53846d0742a828a3ec5430a8a9f80 Reviewed-by: Lars Knoll --- src/plugins/multimedia/ffmpeg/CMakeLists.txt | 5 + src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp | 16 +++- src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp | 46 +++++++++ .../ffmpeg/qffmpeghwaccel_mediacodec.cpp | 106 +++++++++++++++++++++ .../ffmpeg/qffmpeghwaccel_mediacodec_p.h | 71 ++++++++++++++ src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h | 2 + .../multimedia/ffmpeg/qffmpegvideobuffer.cpp | 5 + 7 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp create mode 100644 src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt index 402a3b0aa..9185d61ce 100644 --- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt +++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt @@ -86,8 +86,13 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_linux_v4l if (ANDROID) qt_internal_extend_target(QFFmpegMediaPlugin + SOURCES + qffmpeghwaccel_mediacodec.cpp qffmpeghwaccel_mediacodec_p.h + ../android/wrappers/jni/androidsurfacetexture_p.h + ../android/wrappers/jni/androidsurfacetexture.cpp INCLUDE_DIRECTORIES ${FFMPEG_DIR}/include + ../android/wrappers/jni/ ) set_property(TARGET QFFmpegMediaPlugin APPEND PROPERTY QT_ANDROID_LIB_DEPENDENCIES diff --git a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp index 31b334fbc..3aaf1f781 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp @@ -90,7 +90,9 @@ Codec::Codec(AVFormatContext *format, int streamIndex) Q_ASSERT(streamIndex >= 0 && streamIndex < (int)format->nb_streams); AVStream *stream = format->streams[streamIndex]; - const AVCodec *decoder = avcodec_find_decoder(stream->codecpar->codec_id); + const AVCodec *decoder = + QFFmpeg::HWAccel::hardwareDecoderForCodecId(stream->codecpar->codec_id); + if (!decoder) { qCDebug(qLcDecoder) << "Failed to find a valid FFmpeg decoder"; return; @@ -705,6 +707,18 @@ void VideoRenderer::loop() if (sink) { qint64 startTime = frame.pts(); // qDebug() << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi(); + + // in practice this only happens with mediacodec + if (!frame.codec()->hwAccel().isNull() && !frame.avFrame()->hw_frames_ctx) { + HWAccel hwaccel = frame.codec()->hwAccel(); + AVFrame *avframe = frame.avFrame(); + if (!hwaccel.hwFramesContext()) + hwaccel.createFramesContext(AVPixelFormat(avframe->format), + { avframe->width, avframe->height }); + + avframe->hw_frames_ctx = av_buffer_ref(hwaccel.hwFramesContextAsBuffer()); + } + QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(frame.takeAVFrame()); QVideoFrameFormat format(buffer->size(), buffer->pixelFormat()); format.setColorSpace(buffer->colorSpace()); diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp index e28a89ad8..318db4186 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp @@ -47,6 +47,9 @@ #if QT_CONFIG(wmf) #include "qffmpeghwaccel_d3d11_p.h" #endif +#ifdef Q_OS_ANDROID +# include "qffmpeghwaccel_mediacodec_p.h" +#endif #include "qffmpeg_p.h" #include "qffmpegvideobuffer_p.h" @@ -144,6 +147,10 @@ AVPixelFormat getFormat(AVCodecContext *s, const AVPixelFormat *fmt) #if QT_CONFIG(wmf) if (fmt[n] == AV_PIX_FMT_D3D11) QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(s); +#endif +#ifdef Q_OS_ANDROID + if (fmt[n] == AV_PIX_FMT_MEDIACODEC) + QFFmpeg::MediaCodecTextureConverter::setupDecoderSurface(s); #endif return fmt[n]; } @@ -230,11 +237,45 @@ AVPixelFormat HWAccel::hwFormat() const return AV_PIX_FMT_VIDEOTOOLBOX; case AV_HWDEVICE_TYPE_VAAPI: return AV_PIX_FMT_VAAPI; + case AV_HWDEVICE_TYPE_MEDIACODEC: + return AV_PIX_FMT_MEDIACODEC; default: return AV_PIX_FMT_NONE; } } +const AVCodec *HWAccel::hardwareDecoderForCodecId(AVCodecID id) +{ + const auto getDecoder = [](AVCodecID id) { + switch (id) { +#ifdef Q_OS_ANDROID + case AV_CODEC_ID_H264: + return avcodec_find_decoder_by_name("h264_mediacodec"); + case AV_CODEC_ID_HEVC: + return avcodec_find_decoder_by_name("hevc_mediacodec"); + case AV_CODEC_ID_MPEG2VIDEO: + return avcodec_find_decoder_by_name("mpeg2_mediacodec"); + case AV_CODEC_ID_MPEG4: + return avcodec_find_decoder_by_name("mpeg4_mediacodec"); + case AV_CODEC_ID_VP8: + return avcodec_find_decoder_by_name("vp8_mediacodec"); + case AV_CODEC_ID_VP9: + return avcodec_find_decoder_by_name("vp9_mediacodec"); +#endif + default: + return avcodec_find_decoder(id); + } + }; + + const auto *codec = getDecoder(id); + + // only if ffmpeg build was not setup properly + if (Q_UNLIKELY(!codec)) + codec = avcodec_find_decoder(id); + + return codec; +} + const AVCodec *HWAccel::hardwareEncoderForCodecId(AVCodecID id) const { const char *codec = nullptr; @@ -373,6 +414,11 @@ void TextureConverter::updateBackend(AVPixelFormat fmt) case AV_PIX_FMT_D3D11: d->backend = new D3D11TextureConverter(d->rhi); break; +#endif +#ifdef Q_OS_ANDROID + case AV_PIX_FMT_MEDIACODEC: + d->backend = new MediaCodecTextureConverter(d->rhi); + break; #endif default: break; diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp new file mode 100644 index 000000000..e8872a1eb --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp @@ -0,0 +1,106 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qffmpeghwaccel_mediacodec_p.h" + +#include +#include + +extern "C" { +#include +} + +#if !defined(Q_OS_ANDROID) +# error "Configuration error" +#endif + +namespace QFFmpeg { + +Q_GLOBAL_STATIC(AndroidSurfaceTexture, androidSurfaceTexture, 0); + +class MediaCodecTextureSet : public TextureSet +{ +public: + MediaCodecTextureSet(qint64 textureHandle) : handle(textureHandle) { } + + qint64 textureHandle(int plane) override { return (plane == 0) ? handle : 0; } + +private: + qint64 handle; +}; + +void MediaCodecTextureConverter::setupDecoderSurface(AVCodecContext *avCodecContext) +{ + AVMediaCodecContext *mediacodecContext = av_mediacodec_alloc_context(); + av_mediacodec_default_init(avCodecContext, mediacodecContext, androidSurfaceTexture->surface()); +} + +TextureSet *MediaCodecTextureConverter::getTextures(AVFrame *frame) +{ + if (!androidSurfaceTexture->isValid()) + return {}; + + if (!externalTexture) { + androidSurfaceTexture->detachFromGLContext(); + externalTexture = std::unique_ptr( + rhi->newTexture(QRhiTexture::Format::RGBA8, { frame->width, frame->height }, 1, + QRhiTexture::ExternalOES)); + + if (!externalTexture->create()) { + qWarning() << "Failed to create the external texture!"; + return {}; + } + + quint64 textureHandle = externalTexture->nativeTexture().object; + androidSurfaceTexture->attachToGLContext(textureHandle); + } + + // release a MediaCodec buffer and render it to the surface + AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)frame->data[3]; + int result = av_mediacodec_release_buffer(buffer, 1); + if (result < 0) { + qWarning() << "Failed to render buffer to surface."; + return {}; + } + + androidSurfaceTexture->updateTexImage(); + + return new MediaCodecTextureSet(externalTexture->nativeTexture().object); +} +} diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h new file mode 100644 index 000000000..df6dd091f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h @@ -0,0 +1,71 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QFFMPEGHWACCEL_MEDIACODEC_P_H +#define QFFMPEGHWACCEL_MEDIACODEC_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeghwaccel_p.h" +#include + +namespace QFFmpeg { +struct Frame; + +class MediaCodecTextureConverter : public TextureConverterBackend +{ +public: + MediaCodecTextureConverter(QRhi *rhi) : TextureConverterBackend(rhi){}; + TextureSet *getTextures(AVFrame *frame) override; + + static void setupDecoderSurface(AVCodecContext *s); +private: + std::unique_ptr externalTexture; +}; +} +#endif // QFFMPEGHWACCEL_MEDIACODEC_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h index c25faecfd..2d6acf1f3 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h @@ -142,6 +142,8 @@ public: const AVCodec *hardwareEncoderForCodecId(AVCodecID id) const; static HWAccel findHardwareAccelForCodecID(AVCodecID id); + static const AVCodec *hardwareDecoderForCodecId(AVCodecID id); + void createFramesContext(AVPixelFormat swFormat, const QSize &size); AVBufferRef *hwFramesContextAsBuffer() const { return d ? d->hwFramesContext : nullptr; } AVHWFramesContext *hwFramesContext() const; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp index 144281896..754c84743 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp @@ -309,6 +309,8 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat return QVideoFrameFormat::Format_P010; case AV_PIX_FMT_P016: return QVideoFrameFormat::Format_P016; + case AV_PIX_FMT_MEDIACODEC: + return QVideoFrameFormat::Format_SamplerExternalOES; } if (needsConversion) @@ -382,6 +384,9 @@ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat return AV_PIX_FMT_P010; case QVideoFrameFormat::Format_P016: return AV_PIX_FMT_P016; + + case QVideoFrameFormat::Format_SamplerExternalOES: + return AV_PIX_FMT_MEDIACODEC; } } -- cgit v1.2.3