diff options
13 files changed, 155 insertions, 66 deletions
diff --git a/src/multimedia/video/qabstractvideobuffer.cpp b/src/multimedia/video/qabstractvideobuffer.cpp index f0dd6d2eb..793241be9 100644 --- a/src/multimedia/video/qabstractvideobuffer.cpp +++ b/src/multimedia/video/qabstractvideobuffer.cpp @@ -97,6 +97,7 @@ int QAbstractVideoBufferPrivate::map( \value NoHandle The buffer has no handle, its data can only be accessed by mapping the buffer. \value GLTextureHandle The handle of the buffer is an OpenGL texture ID. + \value MTLTextureHandle The handle of the buffer is an Metal texture ID. \value XvShmImageHandle The handle contains pointer to shared memory XVideo image. \value CoreImageHandle The handle contains pointer to \macos CIImage. \value QPixmapHandle The handle of the buffer is a QPixmap. @@ -363,6 +364,8 @@ QDebug operator<<(QDebug dbg, QAbstractVideoBuffer::HandleType type) return dbg << "NoHandle"; case QAbstractVideoBuffer::GLTextureHandle: return dbg << "GLTextureHandle"; + case QAbstractVideoBuffer::MTLTextureHandle: + return dbg << "MTLTextureHandle"; case QAbstractVideoBuffer::XvShmImageHandle: return dbg << "XvShmImageHandle"; case QAbstractVideoBuffer::CoreImageHandle: diff --git a/src/multimedia/video/qabstractvideobuffer.h b/src/multimedia/video/qabstractvideobuffer.h index 2352c0f3d..b3f31b377 100644 --- a/src/multimedia/video/qabstractvideobuffer.h +++ b/src/multimedia/video/qabstractvideobuffer.h @@ -60,6 +60,7 @@ public: { NoHandle, GLTextureHandle, + MTLTextureHandle, XvShmImageHandle, CoreImageHandle, QPixmapHandle, diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index 99b6bb0b5..d4f74964a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -45,6 +45,9 @@ #include <QtGui/QOpenGLContext> #include <QtCore/QSize> +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class CARenderer; @class AVPlayerLayer; @@ -62,7 +65,8 @@ public: virtual ~AVFVideoFrameRenderer(); - GLuint renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); private: @@ -78,6 +82,9 @@ private: uint m_currentBuffer; bool m_isContextShared; + + id<MTLDevice> m_metalDevice = nil; + id<MTLTexture> m_metalTexture = nil; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 2cdf1cac9..f81412d65 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -72,13 +72,14 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() #endif [m_videoLayerRenderer release]; + [m_metalDevice release]; delete m_fbo[0]; delete m_fbo[1]; delete m_offscreenSurface; delete m_glContext; } -GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) { //Is layer valid if (!layer) @@ -100,6 +101,44 @@ GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) return fbo->texture(); } +quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) +{ + m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); + + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (!m_metalTexture) { + auto desc = [[MTLTextureDescriptor alloc] init]; + desc.textureType = MTLTextureType2D; + desc.width = NSUInteger(m_targetSize.width()); + desc.height = NSUInteger(m_targetSize.height()); + desc.resourceOptions = MTLResourceStorageModePrivate; + desc.usage = MTLTextureUsageRenderTarget; + desc.pixelFormat = MTLPixelFormatRGBA8Unorm; + + m_metalTexture = [m_metalDevice newTextureWithDescriptor: desc]; + [desc release]; + } + + if (!m_videoLayerRenderer) { + m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; + [m_videoLayerRenderer retain]; + } + + if (m_videoLayerRenderer.layer != layer) { + m_videoLayerRenderer.layer = layer; + m_videoLayerRenderer.bounds = layer.bounds; + } + + [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [m_videoLayerRenderer addUpdateRect:layer.bounds]; + [m_videoLayerRenderer render]; + [m_videoLayerRenderer endFrame]; + + return quint64(m_metalTexture); +} + QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) { //Is layer valid @@ -131,8 +170,7 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay : nullptr; //Make sure we have an OpenGL context to make current - if ((shareContext && shareContext != QOpenGLContext::currentContext()) - || (!QOpenGLContext::currentContext() && !m_glContext)) { + if (shareContext || (!QOpenGLContext::currentContext() && !m_glContext)) { //Create Hidden QWindow surface to create context in this thread delete m_offscreenSurface; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h index d9f6baa7e..d1de1f511 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h @@ -45,6 +45,9 @@ #include <QtGui/QOpenGLContext> #include <QtCore/QSize> +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class AVPlayerLayer; @class AVPlayerItemVideoOutput; @@ -92,7 +95,8 @@ public: void setPlayerLayer(AVPlayerLayer *layer); - CVOGLTextureRef renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); private: @@ -106,6 +110,9 @@ private: CVOGLTextureCacheRef m_textureCache; AVPlayerItemVideoOutput* m_videoOutput; bool m_isContextShared; + + id<MTLDevice> m_metalDevice = nil; + CVMetalTextureCacheRef m_metalTextureCache = nil; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm index ed2051449..372b0a27a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm @@ -72,6 +72,9 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() [m_videoOutput release]; // sending to nil is fine if (m_textureCache) CFRelease(m_textureCache); + if (m_metalTextureCache) + CFRelease(m_metalTextureCache); + [m_metalDevice release]; delete m_offscreenSurface; delete m_glContext; } @@ -86,16 +89,59 @@ void AVFVideoFrameRenderer::setPlayerLayer(AVPlayerLayer *layer) } } -CVOGLTextureRef AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) +{ + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (!m_metalTextureCache) { + CVReturn err = CVMetalTextureCacheCreate(kCFAllocatorDefault, nullptr, + m_metalDevice, nullptr, &m_metalTextureCache); + if (err) { + qWarning() << "Error at CVMetalTextureCacheCreate" << err; + return 0; + } + } + + size_t width = 0, height = 0; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + + if (!pixelBuffer) + return 0; + + CVMetalTextureCacheFlush(m_metalTextureCache, 0); + + CVMetalTextureRef texture = nil; + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_metalTextureCache, pixelBuffer, NULL, + MTLPixelFormatBGRA8Unorm_sRGB, width, height, 0, &texture); + + if (!texture || err) + qWarning("CVMetalTextureCacheCreateTextureFromImage failed (error: %d)", err); + + CVPixelBufferRelease(pixelBuffer); + quint64 tex = 0; + if (texture) { + tex = quint64(CVMetalTextureGetTexture(texture)); + CFRelease(texture); + } + + return tex; +} + +quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) { initRenderer(); // If the glContext isn't shared, it doesn't make sense to return a texture for us if (!m_isContextShared) - return nullptr; + return 0; size_t dummyWidth = 0, dummyHeight = 0; - return createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); + auto texture = createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); + auto tex = quint64(CVOGLTextureGetName(texture)); + CFRelease(texture); + + return tex; } static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h index 85dc19d31..c1a629944 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h @@ -84,6 +84,7 @@ private: AVFDisplayLink *m_displayLink; QSize m_nativeSize; bool m_enableOpenGL; + bool m_enableMetal; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index 63bdee4f5..3dbf5e856 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -58,41 +58,11 @@ QT_USE_NAMESPACE -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -class TextureCacheVideoBuffer : public QAbstractVideoBuffer -{ -public: - TextureCacheVideoBuffer(CVOGLTextureRef texture) - : QAbstractVideoBuffer(GLTextureHandle) - , m_texture(texture) - {} - - virtual ~TextureCacheVideoBuffer() - { - // absolutely critical that we drop this - // reference of textures will stay in the cache - CFRelease(m_texture); - } - - MapMode mapMode() const { return NotMapped; } - uchar *map(MapMode, int*, int*) { return nullptr; } - void unmap() {} - - QVariant handle() const - { - GLuint texId = CVOGLTextureGetName(m_texture); - return QVariant::fromValue<unsigned int>(texId); - } - -private: - CVOGLTextureRef m_texture; -}; -#else class TextureVideoBuffer : public QAbstractVideoBuffer { public: - TextureVideoBuffer(GLuint tex) - : QAbstractVideoBuffer(GLTextureHandle) + TextureVideoBuffer(HandleType type, quint64 tex) + : QAbstractVideoBuffer(type) , m_texture(tex) {} @@ -106,13 +76,12 @@ public: QVariant handle() const { - return QVariant::fromValue<unsigned int>(m_texture); + return QVariant::fromValue<unsigned long long>(m_texture); } private: - GLuint m_texture; + quint64 m_texture; }; -#endif AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) @@ -120,6 +89,7 @@ AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) , m_playerLayer(nullptr) , m_frameRenderer(nullptr) , m_enableOpenGL(false) + , m_enableMetal(false) { m_displayLink = new AVFDisplayLink(this); @@ -176,12 +146,12 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) } #endif - //Check for needed formats to render as OpenGL Texture - auto handleGlEnabled = [this] { + auto checkHandleType = [this] { m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); + m_enableMetal = m_surface->supportedPixelFormats(QAbstractVideoBuffer::MTLTextureHandle).contains(QVideoFrame::Format_BGR32); }; - handleGlEnabled(); - connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, handleGlEnabled); + checkHandleType(); + connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, checkHandleType); //If we already have a layer, but changed surfaces start rendering again if (m_playerLayer && !m_displayLink->isActive()) { @@ -235,26 +205,43 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) return; } - if (!playerLayer.readyForDisplay) + if (!playerLayer.readyForDisplay || !m_surface) return; - if (m_enableOpenGL) { -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer); - - //Make sure we got a valid texture - if (tex == nullptr) + if (m_enableMetal) { + quint64 tex = m_frameRenderer->renderLayerToMTLTexture(playerLayer); + if (tex == 0) return; - QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex); + auto buffer = new TextureVideoBuffer(QAbstractVideoBuffer::MTLTextureHandle, tex); + QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); + if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) + m_surface->stop(); + + if (!m_surface->isActive()) { + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::MTLTextureHandle); +#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) + format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); #else - GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer); + format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); +#endif + if (!m_surface->start(format)) + qWarning("Failed to activate video surface"); + } + + if (m_surface->isActive()) + m_surface->present(frame); + + return; + } + + if (m_enableOpenGL) { + quint64 tex = m_frameRenderer->renderLayerToTexture(playerLayer); //Make sure we got a valid texture if (tex == 0) return; - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex); -#endif + QAbstractVideoBuffer *buffer = new TextureVideoBuffer(QAbstractVideoBuffer::GLTextureHandle, tex); QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); if (m_surface && frame.isValid()) { diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index d1dab530f..604866058 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -6,7 +6,7 @@ CONFIG += no_keywords QT += opengl multimedia-private network -LIBS += -framework CoreMedia -framework CoreVideo -framework QuartzCore +LIBS += -framework CoreMedia -framework CoreVideo -framework QuartzCore -framework Metal QMAKE_USE += avfoundation diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp index de7b8efd9..f96d2caf3 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp @@ -38,6 +38,7 @@ ****************************************************************************/ #include "qsgvideonode_texture_p.h" #include "qsgvideotexture_p.h" +#include <private/qsgrhisupport_p.h> #include <QtQuick/qsgmaterial.h> #include <QtCore/qmutex.h> #include <QtMultimedia/private/qtmultimediaglobal_p.h> @@ -48,8 +49,9 @@ QList<QVideoFrame::PixelFormat> QSGVideoNodeFactory_Texture::supportedPixelForma QAbstractVideoBuffer::HandleType handleType) const { QList<QVideoFrame::PixelFormat> pixelFormats; - - if (handleType == QAbstractVideoBuffer::GLTextureHandle) { + auto rhi = QSGRhiSupport::instance(); + auto metalEnabled = rhi->isRhiEnabled() && rhi->rhiBackend() == QRhi::Metal && handleType == QAbstractVideoBuffer::MTLTextureHandle; + if (handleType == QAbstractVideoBuffer::GLTextureHandle || metalEnabled) { pixelFormats.append(QVideoFrame::Format_RGB565); pixelFormats.append(QVideoFrame::Format_RGB32); pixelFormats.append(QVideoFrame::Format_ARGB32); diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h b/src/qtmultimediaquicktools/qsgvideonode_texture_p.h index 12685dd24..d7348473e 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h +++ b/src/qtmultimediaquicktools/qsgvideonode_texture_p.h @@ -68,7 +68,7 @@ public: return m_format.pixelFormat(); } QAbstractVideoBuffer::HandleType handleType() const override { - return QAbstractVideoBuffer::GLTextureHandle; + return m_format.handleType(); } void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; diff --git a/src/qtmultimediaquicktools/qsgvideotexture.cpp b/src/qtmultimediaquicktools/qsgvideotexture.cpp index 0723543ed..bdefbc32f 100644 --- a/src/qtmultimediaquicktools/qsgvideotexture.cpp +++ b/src/qtmultimediaquicktools/qsgvideotexture.cpp @@ -166,9 +166,6 @@ void QSGVideoTexturePrivate::updateRhiTexture(QRhi *rhi, QRhiResourceUpdateBatch QRhiTextureUploadDescription desc({ entry }); resourceUpdates->uploadTexture(m_texture.data(), desc); } - if (q->hasMipmaps()) - resourceUpdates->generateMips(m_texture.data()); - } void QSGVideoTexture::commitTextureOperations(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro index a2451e6e2..82e3e463f 100644 --- a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro +++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro @@ -1,6 +1,6 @@ TARGET = QtMultimediaQuick -QT = core quick multimedia-private +QT = core quick multimedia-private quick-private CONFIG += internal_module PRIVATE_HEADERS += \ |