From 22ae5eec6314b59c8a969b743a9c05fb184cc9b2 Mon Sep 17 00:00:00 2001 From: Val Doroshchuk Date: Thu, 28 May 2020 17:35:16 +0200 Subject: Introduce QAbstractVideoBuffer::MTLTextureHandle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added MTLTextureHandle to render metal textures. Is used by default if rhi is enabled for metal backend. Also fixed the frame renderer to create new opengl context and use provided one from the video surface as a share context. To remember, when the quick item is created and updatePaintNode is called, current gl context is set to the video surface as a property. When the frame renderer is ready, it extracts the gl context and uses it as a share one. Task-number: QTBUG-78678 Change-Id: I51ce666ca7c2adc10dd2c1d1dfed99cc9f596e2b Reviewed-by: Christian Strømme --- src/multimedia/video/qabstractvideobuffer.cpp | 3 + src/multimedia/video/qabstractvideobuffer.h | 1 + .../mediaplayer/avfvideoframerenderer.h | 9 ++- .../mediaplayer/avfvideoframerenderer.mm | 44 ++++++++++- .../mediaplayer/avfvideoframerenderer_ios.h | 9 ++- .../mediaplayer/avfvideoframerenderer_ios.mm | 52 ++++++++++++- .../mediaplayer/avfvideorenderercontrol.h | 1 + .../mediaplayer/avfvideorenderercontrol.mm | 87 +++++++++------------- .../avfoundation/mediaplayer/mediaplayer.pro | 2 +- .../qsgvideonode_texture.cpp | 6 +- .../qsgvideonode_texture_p.h | 2 +- src/qtmultimediaquicktools/qsgvideotexture.cpp | 3 - .../qtmultimediaquicktools.pro | 2 +- 13 files changed, 155 insertions(+), 66 deletions(-) diff --git a/src/multimedia/video/qabstractvideobuffer.cpp b/src/multimedia/video/qabstractvideobuffer.cpp index f0dd6d2eb..793241be9 100644 --- a/src/multimedia/video/qabstractvideobuffer.cpp +++ b/src/multimedia/video/qabstractvideobuffer.cpp @@ -97,6 +97,7 @@ int QAbstractVideoBufferPrivate::map( \value NoHandle The buffer has no handle, its data can only be accessed by mapping the buffer. \value GLTextureHandle The handle of the buffer is an OpenGL texture ID. + \value MTLTextureHandle The handle of the buffer is an Metal texture ID. \value XvShmImageHandle The handle contains pointer to shared memory XVideo image. \value CoreImageHandle The handle contains pointer to \macos CIImage. \value QPixmapHandle The handle of the buffer is a QPixmap. @@ -363,6 +364,8 @@ QDebug operator<<(QDebug dbg, QAbstractVideoBuffer::HandleType type) return dbg << "NoHandle"; case QAbstractVideoBuffer::GLTextureHandle: return dbg << "GLTextureHandle"; + case QAbstractVideoBuffer::MTLTextureHandle: + return dbg << "MTLTextureHandle"; case QAbstractVideoBuffer::XvShmImageHandle: return dbg << "XvShmImageHandle"; case QAbstractVideoBuffer::CoreImageHandle: diff --git a/src/multimedia/video/qabstractvideobuffer.h b/src/multimedia/video/qabstractvideobuffer.h index 2352c0f3d..b3f31b377 100644 --- a/src/multimedia/video/qabstractvideobuffer.h +++ b/src/multimedia/video/qabstractvideobuffer.h @@ -60,6 +60,7 @@ public: { NoHandle, GLTextureHandle, + MTLTextureHandle, XvShmImageHandle, CoreImageHandle, QPixmapHandle, diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index 99b6bb0b5..d4f74964a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -45,6 +45,9 @@ #include #include +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class CARenderer; @class AVPlayerLayer; @@ -62,7 +65,8 @@ public: virtual ~AVFVideoFrameRenderer(); - GLuint renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); private: @@ -78,6 +82,9 @@ private: uint m_currentBuffer; bool m_isContextShared; + + id m_metalDevice = nil; + id m_metalTexture = nil; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 2cdf1cac9..f81412d65 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -72,13 +72,14 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() #endif [m_videoLayerRenderer release]; + [m_metalDevice release]; delete m_fbo[0]; delete m_fbo[1]; delete m_offscreenSurface; delete m_glContext; } -GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) { //Is layer valid if (!layer) @@ -100,6 +101,44 @@ GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) return fbo->texture(); } +quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) +{ + m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); + + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (!m_metalTexture) { + auto desc = [[MTLTextureDescriptor alloc] init]; + desc.textureType = MTLTextureType2D; + desc.width = NSUInteger(m_targetSize.width()); + desc.height = NSUInteger(m_targetSize.height()); + desc.resourceOptions = MTLResourceStorageModePrivate; + desc.usage = MTLTextureUsageRenderTarget; + desc.pixelFormat = MTLPixelFormatRGBA8Unorm; + + m_metalTexture = [m_metalDevice newTextureWithDescriptor: desc]; + [desc release]; + } + + if (!m_videoLayerRenderer) { + m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; + [m_videoLayerRenderer retain]; + } + + if (m_videoLayerRenderer.layer != layer) { + m_videoLayerRenderer.layer = layer; + m_videoLayerRenderer.bounds = layer.bounds; + } + + [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [m_videoLayerRenderer addUpdateRect:layer.bounds]; + [m_videoLayerRenderer render]; + [m_videoLayerRenderer endFrame]; + + return quint64(m_metalTexture); +} + QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) { //Is layer valid @@ -131,8 +170,7 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay : nullptr; //Make sure we have an OpenGL context to make current - if ((shareContext && shareContext != QOpenGLContext::currentContext()) - || (!QOpenGLContext::currentContext() && !m_glContext)) { + if (shareContext || (!QOpenGLContext::currentContext() && !m_glContext)) { //Create Hidden QWindow surface to create context in this thread delete m_offscreenSurface; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h index d9f6baa7e..d1de1f511 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h @@ -45,6 +45,9 @@ #include #include +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class AVPlayerLayer; @class AVPlayerItemVideoOutput; @@ -92,7 +95,8 @@ public: void setPlayerLayer(AVPlayerLayer *layer); - CVOGLTextureRef renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); private: @@ -106,6 +110,9 @@ private: CVOGLTextureCacheRef m_textureCache; AVPlayerItemVideoOutput* m_videoOutput; bool m_isContextShared; + + id m_metalDevice = nil; + CVMetalTextureCacheRef m_metalTextureCache = nil; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm index ed2051449..372b0a27a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm @@ -72,6 +72,9 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() [m_videoOutput release]; // sending to nil is fine if (m_textureCache) CFRelease(m_textureCache); + if (m_metalTextureCache) + CFRelease(m_metalTextureCache); + [m_metalDevice release]; delete m_offscreenSurface; delete m_glContext; } @@ -86,16 +89,59 @@ void AVFVideoFrameRenderer::setPlayerLayer(AVPlayerLayer *layer) } } -CVOGLTextureRef AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) +{ + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (!m_metalTextureCache) { + CVReturn err = CVMetalTextureCacheCreate(kCFAllocatorDefault, nullptr, + m_metalDevice, nullptr, &m_metalTextureCache); + if (err) { + qWarning() << "Error at CVMetalTextureCacheCreate" << err; + return 0; + } + } + + size_t width = 0, height = 0; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + + if (!pixelBuffer) + return 0; + + CVMetalTextureCacheFlush(m_metalTextureCache, 0); + + CVMetalTextureRef texture = nil; + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_metalTextureCache, pixelBuffer, NULL, + MTLPixelFormatBGRA8Unorm_sRGB, width, height, 0, &texture); + + if (!texture || err) + qWarning("CVMetalTextureCacheCreateTextureFromImage failed (error: %d)", err); + + CVPixelBufferRelease(pixelBuffer); + quint64 tex = 0; + if (texture) { + tex = quint64(CVMetalTextureGetTexture(texture)); + CFRelease(texture); + } + + return tex; +} + +quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) { initRenderer(); // If the glContext isn't shared, it doesn't make sense to return a texture for us if (!m_isContextShared) - return nullptr; + return 0; size_t dummyWidth = 0, dummyHeight = 0; - return createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); + auto texture = createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); + auto tex = quint64(CVOGLTextureGetName(texture)); + CFRelease(texture); + + return tex; } static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h index 85dc19d31..c1a629944 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h @@ -84,6 +84,7 @@ private: AVFDisplayLink *m_displayLink; QSize m_nativeSize; bool m_enableOpenGL; + bool m_enableMetal; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index 63bdee4f5..3dbf5e856 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -58,41 +58,11 @@ QT_USE_NAMESPACE -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -class TextureCacheVideoBuffer : public QAbstractVideoBuffer -{ -public: - TextureCacheVideoBuffer(CVOGLTextureRef texture) - : QAbstractVideoBuffer(GLTextureHandle) - , m_texture(texture) - {} - - virtual ~TextureCacheVideoBuffer() - { - // absolutely critical that we drop this - // reference of textures will stay in the cache - CFRelease(m_texture); - } - - MapMode mapMode() const { return NotMapped; } - uchar *map(MapMode, int*, int*) { return nullptr; } - void unmap() {} - - QVariant handle() const - { - GLuint texId = CVOGLTextureGetName(m_texture); - return QVariant::fromValue(texId); - } - -private: - CVOGLTextureRef m_texture; -}; -#else class TextureVideoBuffer : public QAbstractVideoBuffer { public: - TextureVideoBuffer(GLuint tex) - : QAbstractVideoBuffer(GLTextureHandle) + TextureVideoBuffer(HandleType type, quint64 tex) + : QAbstractVideoBuffer(type) , m_texture(tex) {} @@ -106,13 +76,12 @@ public: QVariant handle() const { - return QVariant::fromValue(m_texture); + return QVariant::fromValue(m_texture); } private: - GLuint m_texture; + quint64 m_texture; }; -#endif AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) @@ -120,6 +89,7 @@ AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) , m_playerLayer(nullptr) , m_frameRenderer(nullptr) , m_enableOpenGL(false) + , m_enableMetal(false) { m_displayLink = new AVFDisplayLink(this); @@ -176,12 +146,12 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) } #endif - //Check for needed formats to render as OpenGL Texture - auto handleGlEnabled = [this] { + auto checkHandleType = [this] { m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); + m_enableMetal = m_surface->supportedPixelFormats(QAbstractVideoBuffer::MTLTextureHandle).contains(QVideoFrame::Format_BGR32); }; - handleGlEnabled(); - connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, handleGlEnabled); + checkHandleType(); + connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, checkHandleType); //If we already have a layer, but changed surfaces start rendering again if (m_playerLayer && !m_displayLink->isActive()) { @@ -235,26 +205,43 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) return; } - if (!playerLayer.readyForDisplay) + if (!playerLayer.readyForDisplay || !m_surface) return; - if (m_enableOpenGL) { -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer); - - //Make sure we got a valid texture - if (tex == nullptr) + if (m_enableMetal) { + quint64 tex = m_frameRenderer->renderLayerToMTLTexture(playerLayer); + if (tex == 0) return; - QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex); + auto buffer = new TextureVideoBuffer(QAbstractVideoBuffer::MTLTextureHandle, tex); + QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); + if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) + m_surface->stop(); + + if (!m_surface->isActive()) { + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::MTLTextureHandle); +#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) + format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); #else - GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer); + format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); +#endif + if (!m_surface->start(format)) + qWarning("Failed to activate video surface"); + } + + if (m_surface->isActive()) + m_surface->present(frame); + + return; + } + + if (m_enableOpenGL) { + quint64 tex = m_frameRenderer->renderLayerToTexture(playerLayer); //Make sure we got a valid texture if (tex == 0) return; - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex); -#endif + QAbstractVideoBuffer *buffer = new TextureVideoBuffer(QAbstractVideoBuffer::GLTextureHandle, tex); QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); if (m_surface && frame.isValid()) { diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index d1dab530f..604866058 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -6,7 +6,7 @@ CONFIG += no_keywords QT += opengl multimedia-private network -LIBS += -framework CoreMedia -framework CoreVideo -framework QuartzCore +LIBS += -framework CoreMedia -framework CoreVideo -framework QuartzCore -framework Metal QMAKE_USE += avfoundation diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp index de7b8efd9..f96d2caf3 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp @@ -38,6 +38,7 @@ ****************************************************************************/ #include "qsgvideonode_texture_p.h" #include "qsgvideotexture_p.h" +#include #include #include #include @@ -48,8 +49,9 @@ QList QSGVideoNodeFactory_Texture::supportedPixelForma QAbstractVideoBuffer::HandleType handleType) const { QList pixelFormats; - - if (handleType == QAbstractVideoBuffer::GLTextureHandle) { + auto rhi = QSGRhiSupport::instance(); + auto metalEnabled = rhi->isRhiEnabled() && rhi->rhiBackend() == QRhi::Metal && handleType == QAbstractVideoBuffer::MTLTextureHandle; + if (handleType == QAbstractVideoBuffer::GLTextureHandle || metalEnabled) { pixelFormats.append(QVideoFrame::Format_RGB565); pixelFormats.append(QVideoFrame::Format_RGB32); pixelFormats.append(QVideoFrame::Format_ARGB32); diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h b/src/qtmultimediaquicktools/qsgvideonode_texture_p.h index 12685dd24..d7348473e 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h +++ b/src/qtmultimediaquicktools/qsgvideonode_texture_p.h @@ -68,7 +68,7 @@ public: return m_format.pixelFormat(); } QAbstractVideoBuffer::HandleType handleType() const override { - return QAbstractVideoBuffer::GLTextureHandle; + return m_format.handleType(); } void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; diff --git a/src/qtmultimediaquicktools/qsgvideotexture.cpp b/src/qtmultimediaquicktools/qsgvideotexture.cpp index 0723543ed..bdefbc32f 100644 --- a/src/qtmultimediaquicktools/qsgvideotexture.cpp +++ b/src/qtmultimediaquicktools/qsgvideotexture.cpp @@ -166,9 +166,6 @@ void QSGVideoTexturePrivate::updateRhiTexture(QRhi *rhi, QRhiResourceUpdateBatch QRhiTextureUploadDescription desc({ entry }); resourceUpdates->uploadTexture(m_texture.data(), desc); } - if (q->hasMipmaps()) - resourceUpdates->generateMips(m_texture.data()); - } void QSGVideoTexture::commitTextureOperations(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro index a2451e6e2..82e3e463f 100644 --- a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro +++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro @@ -1,6 +1,6 @@ TARGET = QtMultimediaQuick -QT = core quick multimedia-private +QT = core quick multimedia-private quick-private CONFIG += internal_module PRIVATE_HEADERS += \ -- cgit v1.2.3