diff options
Diffstat (limited to 'src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp')
| -rw-r--r-- | src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp | 392 |
1 files changed, 392 insertions, 0 deletions
diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp new file mode 100644 index 000000000..5d5c67ee0 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp @@ -0,0 +1,392 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qgstvideobuffer_p.h" +#include "qgstreamervideosink_p.h" +#include <private/qvideotexturehelper_p.h> +#include <qpa/qplatformnativeinterface.h> +#include <qguiapplication.h> + +#include <gst/video/video.h> +#include <gst/video/video-frame.h> +#include <gst/video/gstvideometa.h> +#include <gst/pbutils/gstpluginsbaseversion.h> + +#include <common/qgstutils_p.h> + +#if QT_CONFIG(gstreamer_gl) +# include <QtGui/rhi/qrhi.h> +# include <QtGui/qopenglcontext.h> +# include <QtGui/qopenglfunctions.h> +# include <QtGui/qopengl.h> + +# include <gst/gl/gstglconfig.h> +# include <gst/gl/gstglmemory.h> +# include <gst/gl/gstglsyncmeta.h> + +# include <EGL/egl.h> +# include <EGL/eglext.h> + +# if QT_CONFIG(linux_dmabuf) +# include <gst/allocators/gstdmabuf.h> +# endif +#endif + +QT_BEGIN_NAMESPACE + +// keep things building without drm_fourcc.h +#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | \ + ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24)) + +#define DRM_FORMAT_RGBA8888 fourcc_code('R', 'A', '2', '4') /* [31:0] R:G:B:A 8:8:8:8 little endian */ +#define DRM_FORMAT_RGB888 fourcc_code('R', 'G', '2', '4') /* [23:0] R:G:B little endian */ +#define DRM_FORMAT_RG88 fourcc_code('R', 'G', '8', '8') /* [15:0] R:G 8:8 little endian */ +#define DRM_FORMAT_ABGR8888 fourcc_code('A', 'B', '2', '4') /* [31:0] A:B:G:R 8:8:8:8 little endian */ +#define DRM_FORMAT_BGR888 fourcc_code('B', 'G', '2', '4') /* [23:0] B:G:R little endian */ +#define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8') /* [15:0] G:R 8:8 little endian */ +#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ') /* [7:0] R */ +#define DRM_FORMAT_R16 fourcc_code('R', '1', '6', ' ') /* [15:0] R little endian */ +#define DRM_FORMAT_RGB565 fourcc_code('R', 'G', '1', '6') /* [15:0] R:G:B 5:6:5 little endian */ +#define DRM_FORMAT_RG1616 fourcc_code('R', 'G', '3', '2') /* [31:0] R:G 16:16 little endian */ +#define DRM_FORMAT_GR1616 fourcc_code('G', 'R', '3', '2') /* [31:0] G:R 16:16 little endian */ +#define DRM_FORMAT_BGRA1010102 fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */ + +QGstVideoBuffer::QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, + QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, + QGstCaps::MemoryFormat format) + : QAbstractVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) + ? QVideoFrame::RhiTextureHandle + : QVideoFrame::NoHandle, + sink ? sink->rhi() : nullptr), + memoryFormat(format), + m_frameFormat(frameFormat), + m_rhi(sink ? sink->rhi() : nullptr), + m_videoInfo(info), + m_buffer(std::move(buffer)) +{ + if (sink) { + eglDisplay = sink->eglDisplay(); + eglImageTargetTexture2D = sink->eglImageTargetTexture2D(); + } + +#if !QT_CONFIG(gstreamer_gl) + Q_UNUSED(memoryFormat); +#endif +} + +QGstVideoBuffer::~QGstVideoBuffer() +{ + unmap(); +} + +QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) +{ + const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0) + | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0)); + + MapData mapData; + if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped) + return mapData; + + if (m_videoInfo.finfo->n_planes == 0) { // Encoded + if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) { + mapData.nPlanes = 1; + mapData.bytesPerLine[0] = -1; + mapData.size[0] = m_frame.map[0].size; + mapData.data[0] = static_cast<uchar *>(m_frame.map[0].data); + + m_mode = mode; + } + } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer.get(), flags)) { + mapData.nPlanes = GST_VIDEO_FRAME_N_PLANES(&m_frame); + + for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) { + mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i); + mapData.data[i] = static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i)); + mapData.size[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); + } + + m_mode = mode; + } + return mapData; +} + +void QGstVideoBuffer::unmap() +{ + if (m_mode != QVideoFrame::NotMapped) { + if (m_videoInfo.finfo->n_planes == 0) + gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]); + else + gst_video_frame_unmap(&m_frame); + } + m_mode = QVideoFrame::NotMapped; +} + +#if QT_CONFIG(gstreamer_gl) && QT_CONFIG(linux_dmabuf) +static int +fourccFromVideoInfo(const GstVideoInfo * info, int plane) +{ + GstVideoFormat format = GST_VIDEO_INFO_FORMAT (info); +#if G_BYTE_ORDER == G_LITTLE_ENDIAN + const gint rgba_fourcc = DRM_FORMAT_ABGR8888; + const gint rgb_fourcc = DRM_FORMAT_BGR888; + const gint rg_fourcc = DRM_FORMAT_GR88; +#else + const gint rgba_fourcc = DRM_FORMAT_RGBA8888; + const gint rgb_fourcc = DRM_FORMAT_RGB888; + const gint rg_fourcc = DRM_FORMAT_RG88; +#endif + + GST_DEBUG ("Getting DRM fourcc for %s plane %i", + gst_video_format_to_string (format), plane); + + switch (format) { + case GST_VIDEO_FORMAT_RGB16: + case GST_VIDEO_FORMAT_BGR16: + return DRM_FORMAT_RGB565; + + case GST_VIDEO_FORMAT_RGB: + case GST_VIDEO_FORMAT_BGR: + return rgb_fourcc; + + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_ARGB: + case GST_VIDEO_FORMAT_xRGB: + case GST_VIDEO_FORMAT_ABGR: + case GST_VIDEO_FORMAT_xBGR: + case GST_VIDEO_FORMAT_AYUV: +#if GST_CHECK_PLUGINS_BASE_VERSION(1,16,0) + case GST_VIDEO_FORMAT_VUYA: +#endif + return rgba_fourcc; + + case GST_VIDEO_FORMAT_GRAY8: + return DRM_FORMAT_R8; + + case GST_VIDEO_FORMAT_YUY2: + case GST_VIDEO_FORMAT_UYVY: + case GST_VIDEO_FORMAT_GRAY16_LE: + case GST_VIDEO_FORMAT_GRAY16_BE: + return rg_fourcc; + + case GST_VIDEO_FORMAT_NV12: + case GST_VIDEO_FORMAT_NV21: + return plane == 0 ? DRM_FORMAT_R8 : rg_fourcc; + + case GST_VIDEO_FORMAT_I420: + case GST_VIDEO_FORMAT_YV12: + case GST_VIDEO_FORMAT_Y41B: + case GST_VIDEO_FORMAT_Y42B: + case GST_VIDEO_FORMAT_Y444: + return DRM_FORMAT_R8; + +#if GST_CHECK_PLUGINS_BASE_VERSION(1,16,0) + case GST_VIDEO_FORMAT_BGR10A2_LE: + return DRM_FORMAT_BGRA1010102; +#endif + +// case GST_VIDEO_FORMAT_RGB10A2_LE: +// return DRM_FORMAT_RGBA1010102; + + case GST_VIDEO_FORMAT_P010_10LE: +// case GST_VIDEO_FORMAT_P012_LE: +// case GST_VIDEO_FORMAT_P016_LE: + return plane == 0 ? DRM_FORMAT_R16 : DRM_FORMAT_GR1616; + + case GST_VIDEO_FORMAT_P010_10BE: +// case GST_VIDEO_FORMAT_P012_BE: +// case GST_VIDEO_FORMAT_P016_BE: + return plane == 0 ? DRM_FORMAT_R16 : DRM_FORMAT_RG1616; + + default: + GST_ERROR ("Unsupported format for DMABuf."); + return -1; + } +} +#endif + +#if QT_CONFIG(gstreamer_gl) +struct GlTextures +{ + uint count = 0; + bool owned = false; + std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{}; +}; + +class QGstQVideoFrameTextures : public QVideoFrameTextures +{ +public: + QGstQVideoFrameTextures(QRhi *rhi, QSize size, QVideoFrameFormat::PixelFormat format, GlTextures &textures) + : m_rhi(rhi) + , m_glTextures(textures) + { + auto desc = QVideoTextureHelper::textureDescription(format); + for (uint i = 0; i < textures.count; ++i) { + QSize planeSize(desc->widthForPlane(size.width(), int(i)), + desc->heightForPlane(size.height(), int(i))); + m_textures[i].reset(rhi->newTexture(desc->textureFormat[i], planeSize, 1, {})); + m_textures[i]->createFrom({textures.names[i], 0}); + } + } + + ~QGstQVideoFrameTextures() + { + m_rhi->makeThreadLocalNativeContextCurrent(); + auto ctx = QOpenGLContext::currentContext(); + if (m_glTextures.owned && ctx) + ctx->functions()->glDeleteTextures(int(m_glTextures.count), m_glTextures.names.data()); + } + + QRhiTexture *texture(uint plane) const override + { + return plane < m_glTextures.count ? m_textures[plane].get() : nullptr; + } + +private: + QRhi *m_rhi = nullptr; + GlTextures m_glTextures; + std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes]; +}; + +static GlTextures mapFromGlTexture(const QGstBufferHandle &bufferHandle, GstVideoFrame &frame, + GstVideoInfo &videoInfo) +{ + GstBuffer *buffer = bufferHandle.get(); + auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0)); + if (!mem) + return {}; + + if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) { + qWarning() << "Could not map GL textures"; + return {}; + } + + auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer); + GstBuffer *sync_buffer = nullptr; + if (!sync_meta) { + sync_buffer = gst_buffer_new(); + sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer); + } + gst_gl_sync_meta_set_sync_point (sync_meta, mem->context); + gst_gl_sync_meta_wait (sync_meta, mem->context); + if (sync_buffer) + gst_buffer_unref(sync_buffer); + + GlTextures textures; + textures.count = frame.info.finfo->n_planes; + + for (uint i = 0; i < textures.count; ++i) + textures.names[i] = *(guint32 *)frame.data[i]; + + gst_video_frame_unmap(&frame); + + return textures; +} + +#if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) +static GlTextures mapFromDmaBuffer(QRhi *rhi, const QGstBufferHandle &bufferHandle, + GstVideoFrame &frame, GstVideoInfo &videoInfo, + Qt::HANDLE eglDisplay, QFunctionPointer eglImageTargetTexture2D) +{ + GstBuffer *buffer = bufferHandle.get(); + + Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0))); + Q_ASSERT(eglDisplay); + Q_ASSERT(eglImageTargetTexture2D); + + auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); + auto glContext = nativeHandles->context; + if (!glContext) { + qWarning() << "no GL context"; + return {}; + } + + if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ))) { + qDebug() << "Couldn't map DMA video frame"; + return {}; + } + + GlTextures textures = {}; + textures.owned = true; + textures.count = GST_VIDEO_FRAME_N_PLANES(&frame); + // int width = GST_VIDEO_FRAME_WIDTH(&frame); + // int height = GST_VIDEO_FRAME_HEIGHT(&frame); + Q_ASSERT(GST_VIDEO_FRAME_N_PLANES(&frame) == gst_buffer_n_memory(buffer)); + + QOpenGLFunctions functions(glContext); + functions.glGenTextures(int(textures.count), textures.names.data()); + + // qDebug() << Qt::hex << "glGenTextures: glerror" << glGetError() << "egl error" << eglGetError(); + // qDebug() << "converting DMA buffer nPlanes=" << nPlanes << m_textures[0] << m_textures[1] << m_textures[2]; + + for (int i = 0; i < int(textures.count); ++i) { + auto offset = GST_VIDEO_FRAME_PLANE_OFFSET(&frame, i); + auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, i); + int planeWidth = GST_VIDEO_FRAME_COMP_WIDTH(&frame, i); + int planeHeight = GST_VIDEO_FRAME_COMP_HEIGHT(&frame, i); + auto mem = gst_buffer_peek_memory(buffer, i); + int fd = gst_dmabuf_memory_get_fd(mem); + + // qDebug() << " plane" << i << "size" << width << height << "stride" << stride << "offset" << offset << "fd=" << fd; + // ### do we need to open/close the fd? + // ### can we convert several planes at once? + // Get the correct DRM_FORMATs from the texture format in the description + EGLAttrib const attribute_list[] = { + EGL_WIDTH, planeWidth, + EGL_HEIGHT, planeHeight, + EGL_LINUX_DRM_FOURCC_EXT, fourccFromVideoInfo(&videoInfo, i), + EGL_DMA_BUF_PLANE0_FD_EXT, fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLAttrib)offset, + EGL_DMA_BUF_PLANE0_PITCH_EXT, stride, + EGL_NONE + }; + EGLImage image = eglCreateImage(eglDisplay, + EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, + attribute_list); + if (image == EGL_NO_IMAGE_KHR) { + qWarning() << "could not create EGL image for plane" << i << Qt::hex << eglGetError(); + } + // qDebug() << Qt::hex << "eglCreateImage: glerror" << glGetError() << "egl error" << eglGetError(); + functions.glBindTexture(GL_TEXTURE_2D, textures.names[i]); + // qDebug() << Qt::hex << "bind texture: glerror" << glGetError() << "egl error" << eglGetError(); + auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D; + EGLImageTargetTexture2D(GL_TEXTURE_2D, image); + // qDebug() << Qt::hex << "glerror" << glGetError() << "egl error" << eglGetError(); + eglDestroyImage(eglDisplay, image); + } + gst_video_frame_unmap(&frame); + + return textures; +} +#endif +#endif + +std::unique_ptr<QVideoFrameTextures> QGstVideoBuffer::mapTextures(QRhi *rhi) +{ + if (!rhi) + return {}; + +#if QT_CONFIG(gstreamer_gl) + GlTextures textures = {}; + if (memoryFormat == QGstCaps::GLTexture) + textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo); + +# if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) + else if (memoryFormat == QGstCaps::DMABuf) + textures = mapFromDmaBuffer(m_rhi, m_buffer, m_frame, m_videoInfo, eglDisplay, + eglImageTargetTexture2D); + +# endif + if (textures.count > 0) + return std::make_unique<QGstQVideoFrameTextures>(rhi, QSize{m_videoInfo.width, m_videoInfo.height}, + m_frameFormat.pixelFormat(), textures); +#endif + return {}; +} + +QT_END_NAMESPACE |