From 4c2346bbddc83e8a1f8baf9fc335280ed2dace5e Mon Sep 17 00:00:00 2001 From: Andy Nichols Date: Mon, 19 Nov 2012 16:26:38 +0100 Subject: AVFoundation: Enable QImage based frame fallback QGraphicsVideoItem was not working because the QPainterVideoSurface was unable to paint BGR32 format OpenGL textures. Now if the QGraphicsView window has a QGLWidget viewport, we use the GLTextureHandle to render the video, otherwise we fallback to the software QImage rendered case. Task-number: QTBUG-28017 Change-Id: I9304e0a2536f15075ae34cdd509ef24fbc18604e Reviewed-by: Yoann Lopes --- .../mediaplayer/avfvideoframerenderer.mm | 9 +- .../mediaplayer/avfvideorenderercontrol.h | 1 + .../mediaplayer/avfvideorenderercontrol.mm | 100 +++++++++++++++++---- 3 files changed, 89 insertions(+), 21 deletions(-) (limited to 'src/plugins') diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index ea787dc16..294ea32e3 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -131,6 +131,7 @@ GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) return 0; renderLayerToFBO(layer, fbo); + m_glContext->doneCurrent(); return fbo->texture(); } @@ -148,8 +149,10 @@ QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) return QImage(); renderLayerToFBO(layer, fbo); + QImage fboImage = fbo->toImage().mirrored(); + m_glContext->doneCurrent(); - return fbo->toImage(); + return fboImage; } QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *layer) @@ -179,8 +182,8 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay } else { #ifdef QT_DEBUG_AVF qWarning("failed to get Render Thread context"); - m_isContextShared = false; #endif + m_isContextShared = false; } if (!m_glContext->create()) { qWarning("failed to create QOpenGLContext"); @@ -253,6 +256,4 @@ void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFrameb glFinish(); //Rendering needs to be done before passing texture to video frame fbo->release(); - - m_glContext->doneCurrent(); } diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h index 19916bd1a..adee57858 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h @@ -85,6 +85,7 @@ private: AVFVideoFrameRenderer *m_frameRenderer; AVFDisplayLink *m_displayLink; QSize m_nativeSize; + bool m_enableOpenGL; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index e7d99cd00..289675d76 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -75,11 +75,41 @@ private: GLuint m_textureId; }; +class QImageVideoBuffer : public QAbstractVideoBuffer +{ +public: + QImageVideoBuffer(const QImage &image) + : QAbstractVideoBuffer(NoHandle) + , m_image(image) + , m_mode(NotMapped) + { + + } + + MapMode mapMode() const { return m_mode; } + uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) + { + if (mode != NotMapped && m_mode == NotMapped) { + m_mode = mode; + return m_image.bits(); + } else + return 0; + } + + void unmap() { + m_mode = NotMapped; + } +private: + QImage m_image; + MapMode m_mode; +}; + AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) , m_surface(0) , m_playerLayer(0) , m_frameRenderer(0) + , m_enableOpenGL(false) { m_displayLink = new AVFDisplayLink(this); @@ -132,6 +162,9 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) //Surface changed, so we need a new frame renderer m_frameRenderer = new AVFVideoFrameRenderer(m_surface, this); + //Check for needed formats to render as OpenGL Texture + m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); + //If we already have a layer, but changed surfaces start rendering again if (m_playerLayer && !m_displayLink->isActive()) { m_displayLink->start(); @@ -177,31 +210,64 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) if (!playerLayer.readyForDisplay) return; - GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer); + if (m_enableOpenGL) { - //Make sure we got a valid texture - if (textureId == 0) { - qWarning("renderLayerToTexture failed"); - return; - } + GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer); + + //Make sure we got a valid texture + if (textureId == 0) { + qWarning("renderLayerToTexture failed"); + return; + } + + QAbstractVideoBuffer *buffer = new TextureVideoBuffer(textureId); + QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); + + if (m_surface && frame.isValid()) { + if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) + m_surface->stop(); + + if (!m_surface->isActive()) { + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::GLTextureHandle); + format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(textureId); - QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); + if (!m_surface->start(format)) { + //Surface doesn't support GLTextureHandle + qWarning("Failed to activate video surface"); + } + } + + if (m_surface->isActive()) + m_surface->present(frame); + } + } else { + //fallback to rendering frames to QImages + QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer); - if (m_surface && frame.isValid()) { - if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) - m_surface->stop(); + if (frameData.isNull()) { + qWarning("renterLayerToImage failed"); + return; + } - if (!m_surface->isActive()) { - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::GLTextureHandle); + QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData); + QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32_Premultiplied); - if (!m_surface->start(format)) { - qWarning("Failed to activate video surface"); + if (m_surface && frame.isValid()) { + if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) + m_surface->stop(); + + if (!m_surface->isActive()) { + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::NoHandle); + + if (!m_surface->start(format)) { + qWarning("Failed to activate video surface"); + } } + + if (m_surface->isActive()) + m_surface->present(frame); } - if (m_surface->isActive()) - m_surface->present(frame); } } -- cgit v1.2.3