summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSamuel Mira <samuel.mira@qt.io>2022-05-16 20:16:22 +0300
committerSamuel Mira <samuel.mira@qt.io>2022-05-23 13:08:31 +0000
commit4b8b5ad1bd74fd107f034d72c6d44faa386f01f5 (patch)
tree93994192bc3a99d4345e51d34854c57665e48a3f
parent2835377e2b0be50e4ed13876869c008340feac11 (diff)
Android: Enable FFmpeg playback - HW renderingHEADdev
This patch enables video playback on Android using Hardware rendering with FFmpeg backend. Task-number: QTBUG-102233 Change-Id: Ib9af89ef78e53846d0742a828a3ec5430a8a9f80 Reviewed-by: Lars Knoll <lars.knoll@qt.io>
-rw-r--r--src/plugins/multimedia/ffmpeg/CMakeLists.txt5
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp16
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp46
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp106
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h71
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h2
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp5
7 files changed, 250 insertions, 1 deletions
diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
index 402a3b0aa..9185d61ce 100644
--- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt
+++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
@@ -86,8 +86,13 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_linux_v4l
if (ANDROID)
qt_internal_extend_target(QFFmpegMediaPlugin
+ SOURCES
+ qffmpeghwaccel_mediacodec.cpp qffmpeghwaccel_mediacodec_p.h
+ ../android/wrappers/jni/androidsurfacetexture_p.h
+ ../android/wrappers/jni/androidsurfacetexture.cpp
INCLUDE_DIRECTORIES
${FFMPEG_DIR}/include
+ ../android/wrappers/jni/
)
set_property(TARGET QFFmpegMediaPlugin APPEND PROPERTY QT_ANDROID_LIB_DEPENDENCIES
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp
index 31b334fbc..3aaf1f781 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp
@@ -90,7 +90,9 @@ Codec::Codec(AVFormatContext *format, int streamIndex)
Q_ASSERT(streamIndex >= 0 && streamIndex < (int)format->nb_streams);
AVStream *stream = format->streams[streamIndex];
- const AVCodec *decoder = avcodec_find_decoder(stream->codecpar->codec_id);
+ const AVCodec *decoder =
+ QFFmpeg::HWAccel::hardwareDecoderForCodecId(stream->codecpar->codec_id);
+
if (!decoder) {
qCDebug(qLcDecoder) << "Failed to find a valid FFmpeg decoder";
return;
@@ -705,6 +707,18 @@ void VideoRenderer::loop()
if (sink) {
qint64 startTime = frame.pts();
// qDebug() << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi();
+
+ // in practice this only happens with mediacodec
+ if (!frame.codec()->hwAccel().isNull() && !frame.avFrame()->hw_frames_ctx) {
+ HWAccel hwaccel = frame.codec()->hwAccel();
+ AVFrame *avframe = frame.avFrame();
+ if (!hwaccel.hwFramesContext())
+ hwaccel.createFramesContext(AVPixelFormat(avframe->format),
+ { avframe->width, avframe->height });
+
+ avframe->hw_frames_ctx = av_buffer_ref(hwaccel.hwFramesContextAsBuffer());
+ }
+
QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(frame.takeAVFrame());
QVideoFrameFormat format(buffer->size(), buffer->pixelFormat());
format.setColorSpace(buffer->colorSpace());
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
index e28a89ad8..318db4186 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
@@ -47,6 +47,9 @@
#if QT_CONFIG(wmf)
#include "qffmpeghwaccel_d3d11_p.h"
#endif
+#ifdef Q_OS_ANDROID
+# include "qffmpeghwaccel_mediacodec_p.h"
+#endif
#include "qffmpeg_p.h"
#include "qffmpegvideobuffer_p.h"
@@ -145,6 +148,10 @@ AVPixelFormat getFormat(AVCodecContext *s, const AVPixelFormat *fmt)
if (fmt[n] == AV_PIX_FMT_D3D11)
QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(s);
#endif
+#ifdef Q_OS_ANDROID
+ if (fmt[n] == AV_PIX_FMT_MEDIACODEC)
+ QFFmpeg::MediaCodecTextureConverter::setupDecoderSurface(s);
+#endif
return fmt[n];
}
}
@@ -230,11 +237,45 @@ AVPixelFormat HWAccel::hwFormat() const
return AV_PIX_FMT_VIDEOTOOLBOX;
case AV_HWDEVICE_TYPE_VAAPI:
return AV_PIX_FMT_VAAPI;
+ case AV_HWDEVICE_TYPE_MEDIACODEC:
+ return AV_PIX_FMT_MEDIACODEC;
default:
return AV_PIX_FMT_NONE;
}
}
+const AVCodec *HWAccel::hardwareDecoderForCodecId(AVCodecID id)
+{
+ const auto getDecoder = [](AVCodecID id) {
+ switch (id) {
+#ifdef Q_OS_ANDROID
+ case AV_CODEC_ID_H264:
+ return avcodec_find_decoder_by_name("h264_mediacodec");
+ case AV_CODEC_ID_HEVC:
+ return avcodec_find_decoder_by_name("hevc_mediacodec");
+ case AV_CODEC_ID_MPEG2VIDEO:
+ return avcodec_find_decoder_by_name("mpeg2_mediacodec");
+ case AV_CODEC_ID_MPEG4:
+ return avcodec_find_decoder_by_name("mpeg4_mediacodec");
+ case AV_CODEC_ID_VP8:
+ return avcodec_find_decoder_by_name("vp8_mediacodec");
+ case AV_CODEC_ID_VP9:
+ return avcodec_find_decoder_by_name("vp9_mediacodec");
+#endif
+ default:
+ return avcodec_find_decoder(id);
+ }
+ };
+
+ const auto *codec = getDecoder(id);
+
+ // only if ffmpeg build was not setup properly
+ if (Q_UNLIKELY(!codec))
+ codec = avcodec_find_decoder(id);
+
+ return codec;
+}
+
const AVCodec *HWAccel::hardwareEncoderForCodecId(AVCodecID id) const
{
const char *codec = nullptr;
@@ -374,6 +415,11 @@ void TextureConverter::updateBackend(AVPixelFormat fmt)
d->backend = new D3D11TextureConverter(d->rhi);
break;
#endif
+#ifdef Q_OS_ANDROID
+ case AV_PIX_FMT_MEDIACODEC:
+ d->backend = new MediaCodecTextureConverter(d->rhi);
+ break;
+#endif
default:
break;
}
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp
new file mode 100644
index 000000000..e8872a1eb
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp
@@ -0,0 +1,106 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qffmpeghwaccel_mediacodec_p.h"
+
+#include <androidsurfacetexture_p.h>
+#include <QtGui/private/qrhi_p.h>
+
+extern "C" {
+#include <libavcodec/mediacodec.h>
+}
+
+#if !defined(Q_OS_ANDROID)
+# error "Configuration error"
+#endif
+
+namespace QFFmpeg {
+
+Q_GLOBAL_STATIC(AndroidSurfaceTexture, androidSurfaceTexture, 0);
+
+class MediaCodecTextureSet : public TextureSet
+{
+public:
+ MediaCodecTextureSet(qint64 textureHandle) : handle(textureHandle) { }
+
+ qint64 textureHandle(int plane) override { return (plane == 0) ? handle : 0; }
+
+private:
+ qint64 handle;
+};
+
+void MediaCodecTextureConverter::setupDecoderSurface(AVCodecContext *avCodecContext)
+{
+ AVMediaCodecContext *mediacodecContext = av_mediacodec_alloc_context();
+ av_mediacodec_default_init(avCodecContext, mediacodecContext, androidSurfaceTexture->surface());
+}
+
+TextureSet *MediaCodecTextureConverter::getTextures(AVFrame *frame)
+{
+ if (!androidSurfaceTexture->isValid())
+ return {};
+
+ if (!externalTexture) {
+ androidSurfaceTexture->detachFromGLContext();
+ externalTexture = std::unique_ptr<QRhiTexture>(
+ rhi->newTexture(QRhiTexture::Format::RGBA8, { frame->width, frame->height }, 1,
+ QRhiTexture::ExternalOES));
+
+ if (!externalTexture->create()) {
+ qWarning() << "Failed to create the external texture!";
+ return {};
+ }
+
+ quint64 textureHandle = externalTexture->nativeTexture().object;
+ androidSurfaceTexture->attachToGLContext(textureHandle);
+ }
+
+ // release a MediaCodec buffer and render it to the surface
+ AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)frame->data[3];
+ int result = av_mediacodec_release_buffer(buffer, 1);
+ if (result < 0) {
+ qWarning() << "Failed to render buffer to surface.";
+ return {};
+ }
+
+ androidSurfaceTexture->updateTexImage();
+
+ return new MediaCodecTextureSet(externalTexture->nativeTexture().object);
+}
+}
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h
new file mode 100644
index 000000000..df6dd091f
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h
@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QFFMPEGHWACCEL_MEDIACODEC_P_H
+#define QFFMPEGHWACCEL_MEDIACODEC_P_H
+
+//
+// W A R N I N G
+// -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "qffmpeghwaccel_p.h"
+#include <memory>
+
+namespace QFFmpeg {
+struct Frame;
+
+class MediaCodecTextureConverter : public TextureConverterBackend
+{
+public:
+ MediaCodecTextureConverter(QRhi *rhi) : TextureConverterBackend(rhi){};
+ TextureSet *getTextures(AVFrame *frame) override;
+
+ static void setupDecoderSurface(AVCodecContext *s);
+private:
+ std::unique_ptr<QRhiTexture> externalTexture;
+};
+}
+#endif // QFFMPEGHWACCEL_MEDIACODEC_P_H
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
index c25faecfd..2d6acf1f3 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
@@ -142,6 +142,8 @@ public:
const AVCodec *hardwareEncoderForCodecId(AVCodecID id) const;
static HWAccel findHardwareAccelForCodecID(AVCodecID id);
+ static const AVCodec *hardwareDecoderForCodecId(AVCodecID id);
+
void createFramesContext(AVPixelFormat swFormat, const QSize &size);
AVBufferRef *hwFramesContextAsBuffer() const { return d ? d->hwFramesContext : nullptr; }
AVHWFramesContext *hwFramesContext() const;
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index 144281896..754c84743 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -309,6 +309,8 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat
return QVideoFrameFormat::Format_P010;
case AV_PIX_FMT_P016:
return QVideoFrameFormat::Format_P016;
+ case AV_PIX_FMT_MEDIACODEC:
+ return QVideoFrameFormat::Format_SamplerExternalOES;
}
if (needsConversion)
@@ -382,6 +384,9 @@ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat
return AV_PIX_FMT_P010;
case QVideoFrameFormat::Format_P016:
return AV_PIX_FMT_P016;
+
+ case QVideoFrameFormat::Format_SamplerExternalOES:
+ return AV_PIX_FMT_MEDIACODEC;
}
}