diff options
Diffstat (limited to 'src/plugins/avfoundation/mediaplayer')
7 files changed, 145 insertions, 59 deletions
diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index 99b6bb0b5..d4f74964a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -45,6 +45,9 @@ #include <QtGui/QOpenGLContext> #include <QtCore/QSize> +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class CARenderer; @class AVPlayerLayer; @@ -62,7 +65,8 @@ public: virtual ~AVFVideoFrameRenderer(); - GLuint renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); private: @@ -78,6 +82,9 @@ private: uint m_currentBuffer; bool m_isContextShared; + + id<MTLDevice> m_metalDevice = nil; + id<MTLTexture> m_metalTexture = nil; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 2cdf1cac9..f81412d65 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -72,13 +72,14 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() #endif [m_videoLayerRenderer release]; + [m_metalDevice release]; delete m_fbo[0]; delete m_fbo[1]; delete m_offscreenSurface; delete m_glContext; } -GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) { //Is layer valid if (!layer) @@ -100,6 +101,44 @@ GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) return fbo->texture(); } +quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) +{ + m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); + + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (!m_metalTexture) { + auto desc = [[MTLTextureDescriptor alloc] init]; + desc.textureType = MTLTextureType2D; + desc.width = NSUInteger(m_targetSize.width()); + desc.height = NSUInteger(m_targetSize.height()); + desc.resourceOptions = MTLResourceStorageModePrivate; + desc.usage = MTLTextureUsageRenderTarget; + desc.pixelFormat = MTLPixelFormatRGBA8Unorm; + + m_metalTexture = [m_metalDevice newTextureWithDescriptor: desc]; + [desc release]; + } + + if (!m_videoLayerRenderer) { + m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; + [m_videoLayerRenderer retain]; + } + + if (m_videoLayerRenderer.layer != layer) { + m_videoLayerRenderer.layer = layer; + m_videoLayerRenderer.bounds = layer.bounds; + } + + [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [m_videoLayerRenderer addUpdateRect:layer.bounds]; + [m_videoLayerRenderer render]; + [m_videoLayerRenderer endFrame]; + + return quint64(m_metalTexture); +} + QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) { //Is layer valid @@ -131,8 +170,7 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay : nullptr; //Make sure we have an OpenGL context to make current - if ((shareContext && shareContext != QOpenGLContext::currentContext()) - || (!QOpenGLContext::currentContext() && !m_glContext)) { + if (shareContext || (!QOpenGLContext::currentContext() && !m_glContext)) { //Create Hidden QWindow surface to create context in this thread delete m_offscreenSurface; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h index d9f6baa7e..d1de1f511 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h @@ -45,6 +45,9 @@ #include <QtGui/QOpenGLContext> #include <QtCore/QSize> +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class AVPlayerLayer; @class AVPlayerItemVideoOutput; @@ -92,7 +95,8 @@ public: void setPlayerLayer(AVPlayerLayer *layer); - CVOGLTextureRef renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToTexture(AVPlayerLayer *layer); + quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); private: @@ -106,6 +110,9 @@ private: CVOGLTextureCacheRef m_textureCache; AVPlayerItemVideoOutput* m_videoOutput; bool m_isContextShared; + + id<MTLDevice> m_metalDevice = nil; + CVMetalTextureCacheRef m_metalTextureCache = nil; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm index ed2051449..372b0a27a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm @@ -72,6 +72,9 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() [m_videoOutput release]; // sending to nil is fine if (m_textureCache) CFRelease(m_textureCache); + if (m_metalTextureCache) + CFRelease(m_metalTextureCache); + [m_metalDevice release]; delete m_offscreenSurface; delete m_glContext; } @@ -86,16 +89,59 @@ void AVFVideoFrameRenderer::setPlayerLayer(AVPlayerLayer *layer) } } -CVOGLTextureRef AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) +{ + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (!m_metalTextureCache) { + CVReturn err = CVMetalTextureCacheCreate(kCFAllocatorDefault, nullptr, + m_metalDevice, nullptr, &m_metalTextureCache); + if (err) { + qWarning() << "Error at CVMetalTextureCacheCreate" << err; + return 0; + } + } + + size_t width = 0, height = 0; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + + if (!pixelBuffer) + return 0; + + CVMetalTextureCacheFlush(m_metalTextureCache, 0); + + CVMetalTextureRef texture = nil; + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_metalTextureCache, pixelBuffer, NULL, + MTLPixelFormatBGRA8Unorm_sRGB, width, height, 0, &texture); + + if (!texture || err) + qWarning("CVMetalTextureCacheCreateTextureFromImage failed (error: %d)", err); + + CVPixelBufferRelease(pixelBuffer); + quint64 tex = 0; + if (texture) { + tex = quint64(CVMetalTextureGetTexture(texture)); + CFRelease(texture); + } + + return tex; +} + +quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) { initRenderer(); // If the glContext isn't shared, it doesn't make sense to return a texture for us if (!m_isContextShared) - return nullptr; + return 0; size_t dummyWidth = 0, dummyHeight = 0; - return createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); + auto texture = createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); + auto tex = quint64(CVOGLTextureGetName(texture)); + CFRelease(texture); + + return tex; } static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h index 85dc19d31..c1a629944 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h @@ -84,6 +84,7 @@ private: AVFDisplayLink *m_displayLink; QSize m_nativeSize; bool m_enableOpenGL; + bool m_enableMetal; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index 63bdee4f5..3dbf5e856 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -58,41 +58,11 @@ QT_USE_NAMESPACE -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -class TextureCacheVideoBuffer : public QAbstractVideoBuffer -{ -public: - TextureCacheVideoBuffer(CVOGLTextureRef texture) - : QAbstractVideoBuffer(GLTextureHandle) - , m_texture(texture) - {} - - virtual ~TextureCacheVideoBuffer() - { - // absolutely critical that we drop this - // reference of textures will stay in the cache - CFRelease(m_texture); - } - - MapMode mapMode() const { return NotMapped; } - uchar *map(MapMode, int*, int*) { return nullptr; } - void unmap() {} - - QVariant handle() const - { - GLuint texId = CVOGLTextureGetName(m_texture); - return QVariant::fromValue<unsigned int>(texId); - } - -private: - CVOGLTextureRef m_texture; -}; -#else class TextureVideoBuffer : public QAbstractVideoBuffer { public: - TextureVideoBuffer(GLuint tex) - : QAbstractVideoBuffer(GLTextureHandle) + TextureVideoBuffer(HandleType type, quint64 tex) + : QAbstractVideoBuffer(type) , m_texture(tex) {} @@ -106,13 +76,12 @@ public: QVariant handle() const { - return QVariant::fromValue<unsigned int>(m_texture); + return QVariant::fromValue<unsigned long long>(m_texture); } private: - GLuint m_texture; + quint64 m_texture; }; -#endif AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) @@ -120,6 +89,7 @@ AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) , m_playerLayer(nullptr) , m_frameRenderer(nullptr) , m_enableOpenGL(false) + , m_enableMetal(false) { m_displayLink = new AVFDisplayLink(this); @@ -176,12 +146,12 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) } #endif - //Check for needed formats to render as OpenGL Texture - auto handleGlEnabled = [this] { + auto checkHandleType = [this] { m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); + m_enableMetal = m_surface->supportedPixelFormats(QAbstractVideoBuffer::MTLTextureHandle).contains(QVideoFrame::Format_BGR32); }; - handleGlEnabled(); - connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, handleGlEnabled); + checkHandleType(); + connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, checkHandleType); //If we already have a layer, but changed surfaces start rendering again if (m_playerLayer && !m_displayLink->isActive()) { @@ -235,26 +205,43 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) return; } - if (!playerLayer.readyForDisplay) + if (!playerLayer.readyForDisplay || !m_surface) return; - if (m_enableOpenGL) { -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer); - - //Make sure we got a valid texture - if (tex == nullptr) + if (m_enableMetal) { + quint64 tex = m_frameRenderer->renderLayerToMTLTexture(playerLayer); + if (tex == 0) return; - QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex); + auto buffer = new TextureVideoBuffer(QAbstractVideoBuffer::MTLTextureHandle, tex); + QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); + if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) + m_surface->stop(); + + if (!m_surface->isActive()) { + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::MTLTextureHandle); +#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) + format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); #else - GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer); + format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); +#endif + if (!m_surface->start(format)) + qWarning("Failed to activate video surface"); + } + + if (m_surface->isActive()) + m_surface->present(frame); + + return; + } + + if (m_enableOpenGL) { + quint64 tex = m_frameRenderer->renderLayerToTexture(playerLayer); //Make sure we got a valid texture if (tex == 0) return; - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex); -#endif + QAbstractVideoBuffer *buffer = new TextureVideoBuffer(QAbstractVideoBuffer::GLTextureHandle, tex); QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); if (m_surface && frame.isValid()) { diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index d1dab530f..604866058 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -6,7 +6,7 @@ CONFIG += no_keywords QT += opengl multimedia-private network -LIBS += -framework CoreMedia -framework CoreVideo -framework QuartzCore +LIBS += -framework CoreMedia -framework CoreVideo -framework QuartzCore -framework Metal QMAKE_USE += avfoundation |