From 9444c8ec6161bee3e399f9d385974dbaa6216283 Mon Sep 17 00:00:00 2001 From: James Turner Date: Mon, 12 Jan 2015 13:51:25 +0000 Subject: iOS video frame render implementation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses CVTextureCache, iOS only for now, OS-X code could be ported but will need further work to support TEXTURE_RECTANGLE in the QVideoNode classes. When we can’t share a context, falls back to an offscreen window, FBO rendering and grabbing a QImage. Change-Id: I23b831fdcc63aeb1b67b7741d8d56779470240d3 Reviewed-by: Yoann Lopes --- .../mediaplayer/avfvideorenderercontrol.mm | 105 +++++++++++++-------- 1 file changed, 67 insertions(+), 38 deletions(-) (limited to 'src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm') diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index c7087b612..3ca9c1873 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -41,26 +41,40 @@ #include "avfvideorenderercontrol.h" #include "avfdisplaylink.h" + +#if defined(Q_OS_IOS) +#include "avfvideoframerenderer_ios.h" +#else #include "avfvideoframerenderer.h" +#endif #include #include #include + +#include + #include #import QT_USE_NAMESPACE -class TextureVideoBuffer : public QAbstractVideoBuffer +#if defined(Q_OS_IOS) +class TextureCacheVideoBuffer : public QAbstractVideoBuffer { public: - TextureVideoBuffer(GLuint textureId) + TextureCacheVideoBuffer(CVOGLTextureRef texture) : QAbstractVideoBuffer(GLTextureHandle) - , m_textureId(textureId) + , m_texture(texture) {} - virtual ~TextureVideoBuffer() {} + virtual ~TextureCacheVideoBuffer() + { + // absolutely critical that we drop this + // reference of textures will stay in the cache + CFRelease(m_texture); + } MapMode mapMode() const { return NotMapped; } uchar *map(MapMode, int*, int*) { return 0; } @@ -68,41 +82,39 @@ public: QVariant handle() const { - return QVariant::fromValue(m_textureId); + GLuint texId = CVOGLTextureGetName(m_texture); + return QVariant::fromValue(texId); } private: - GLuint m_textureId; + CVOGLTextureRef m_texture; }; - -class QImageVideoBuffer : public QAbstractVideoBuffer +#else +class TextureVideoBuffer : public QAbstractVideoBuffer { public: - QImageVideoBuffer(const QImage &image) - : QAbstractVideoBuffer(NoHandle) - , m_image(image) - , m_mode(NotMapped) - { + TextureVideoBuffer(GLuint tex) + : QAbstractVideoBuffer(GLTextureHandle) + , m_texture(tex) + {} + virtual ~TextureVideoBuffer() + { } - MapMode mapMode() const { return m_mode; } - uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) + MapMode mapMode() const { return NotMapped; } + uchar *map(MapMode, int*, int*) { return 0; } + void unmap() {} + + QVariant handle() const { - if (mode != NotMapped && m_mode == NotMapped) { - m_mode = mode; - return m_image.bits(); - } else - return 0; + return QVariant::fromValue(m_texture); } - void unmap() { - m_mode = NotMapped; - } private: - QImage m_image; - MapMode m_mode; + GLuint m_texture; }; +#endif AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) @@ -122,8 +134,7 @@ AVFVideoRendererControl::~AVFVideoRendererControl() qDebug() << Q_FUNC_INFO; #endif m_displayLink->stop(); - if (m_playerLayer) - [(AVPlayerLayer*)m_playerLayer release]; + [(AVPlayerLayer*)m_playerLayer release]; } QAbstractVideoSurface *AVFVideoRendererControl::surface() const @@ -150,8 +161,8 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) m_surface = surface; //If the surface changed, then the current frame renderer is no longer valid - if (m_frameRenderer) - delete m_frameRenderer; + delete m_frameRenderer; + m_frameRenderer = 0; //If there is now no surface to render too if (m_surface == 0) { @@ -161,6 +172,11 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) //Surface changed, so we need a new frame renderer m_frameRenderer = new AVFVideoFrameRenderer(m_surface, this); +#if defined(Q_OS_IOS) + if (m_playerLayer) { + m_frameRenderer->setPlayerLayer(static_cast(m_playerLayer)); + } +#endif //Check for needed formats to render as OpenGL Texture m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); @@ -187,6 +203,12 @@ void AVFVideoRendererControl::setLayer(void *playerLayer) if (m_surface && m_surface->isActive()) m_surface->stop(); +#if defined(Q_OS_IOS) + if (m_frameRenderer) { + m_frameRenderer->setPlayerLayer(static_cast(playerLayer)); + } +#endif + //If there is no layer to render, stop scheduling updates if (m_playerLayer == 0) { m_displayLink->stop(); @@ -216,16 +238,22 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) return; if (m_enableOpenGL) { +#if defined(Q_OS_IOS) + CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer); - GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer); + //Make sure we got a valid texture + if (tex == 0) + return; + QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex); +#else + GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer); //Make sure we got a valid texture - if (textureId == 0) { - qWarning("renderLayerToTexture failed"); + if (tex == 0) return; - } - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(textureId); + QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex); +#endif QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); if (m_surface && frame.isValid()) { @@ -234,8 +262,11 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) if (!m_surface->isActive()) { QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::GLTextureHandle); +#if defined(Q_OS_IOS) + format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); +#else format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); - +#endif if (!m_surface->start(format)) { //Surface doesn't support GLTextureHandle qWarning("Failed to activate video surface"); @@ -250,13 +281,11 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer); if (frameData.isNull()) { - qWarning("renterLayerToImage failed"); return; } QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData); - QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32_Premultiplied); - + QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32); if (m_surface && frame.isValid()) { if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) m_surface->stop(); -- cgit v1.2.3