From 9444c8ec6161bee3e399f9d385974dbaa6216283 Mon Sep 17 00:00:00 2001 From: James Turner Date: Mon, 12 Jan 2015 13:51:25 +0000 Subject: iOS video frame render implementation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses CVTextureCache, iOS only for now, OS-X code could be ported but will need further work to support TEXTURE_RECTANGLE in the QVideoNode classes. When we can’t share a context, falls back to an offscreen window, FBO rendering and grabbing a QImage. Change-Id: I23b831fdcc63aeb1b67b7741d8d56779470240d3 Reviewed-by: Yoann Lopes --- .../avfoundation/mediaplayer/avfdisplaylink.h | 8 + .../avfoundation/mediaplayer/avfdisplaylink.mm | 92 +++++++- .../mediaplayer/avfmediaplayerservice.h | 1 + .../mediaplayer/avfmediaplayerservice.mm | 32 ++- .../mediaplayer/avfvideoframerenderer_ios.h | 107 +++++++++ .../mediaplayer/avfvideoframerenderer_ios.mm | 261 +++++++++++++++++++++ .../mediaplayer/avfvideorenderercontrol.mm | 105 ++++++--- .../avfoundation/mediaplayer/mediaplayer.pro | 30 ++- 8 files changed, 575 insertions(+), 61 deletions(-) create mode 100644 src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h create mode 100644 src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm (limited to 'src/plugins/avfoundation/mediaplayer') diff --git a/src/plugins/avfoundation/mediaplayer/avfdisplaylink.h b/src/plugins/avfoundation/mediaplayer/avfdisplaylink.h index 3a4b4af9e..ea1dd219a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfdisplaylink.h +++ b/src/plugins/avfoundation/mediaplayer/avfdisplaylink.h @@ -37,7 +37,11 @@ #include #include +#if defined(Q_OS_IOS) +#include +#else #include +#endif QT_BEGIN_NAMESPACE @@ -64,7 +68,11 @@ protected: virtual bool event(QEvent *); private: +#if defined(Q_OS_IOS) + void *m_displayLink; +#else CVDisplayLinkRef m_displayLink; +#endif QMutex m_displayLinkMutex; bool m_pendingDisplayLinkEvent; bool m_isActive; diff --git a/src/plugins/avfoundation/mediaplayer/avfdisplaylink.mm b/src/plugins/avfoundation/mediaplayer/avfdisplaylink.mm index b3f9b690d..d52a4a7fe 100644 --- a/src/plugins/avfoundation/mediaplayer/avfdisplaylink.mm +++ b/src/plugins/avfoundation/mediaplayer/avfdisplaylink.mm @@ -46,8 +46,70 @@ #include #endif +#if defined(Q_OS_IOS) +#import +#import +#define _m_displayLink static_cast(m_displayLink) +#else +#endif + QT_USE_NAMESPACE +#if defined(Q_OS_IOS) +@interface DisplayLinkObserver : NSObject +{ + AVFDisplayLink *m_avfDisplayLink; + CADisplayLink *m_displayLink; +} + +- (void)start; +- (void)stop; +- (void)displayLinkNotification:(CADisplayLink *)sender; + +@end + +@implementation DisplayLinkObserver + +- (id)initWithAVFDisplayLink:(AVFDisplayLink *)link +{ + self = [super init]; + + if (self) { + m_avfDisplayLink = link; + m_displayLink = [[CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkNotification:)] retain]; + } + + return self; +} + +- (void) dealloc +{ + if (m_displayLink) { + [m_displayLink release]; + m_displayLink = NULL; + } + + [super dealloc]; +} + +- (void)start +{ + [m_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; +} + +- (void)stop +{ + [m_displayLink removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; +} + +- (void)displayLinkNotification:(CADisplayLink *)sender +{ + Q_UNUSED(sender); + m_avfDisplayLink->displayLinkEvent(nullptr); +} + +@end +#else static CVReturn CVDisplayLinkCallback(CVDisplayLinkRef displayLink, const CVTimeStamp *inNow, const CVTimeStamp *inOutputTime, @@ -65,12 +127,17 @@ static CVReturn CVDisplayLinkCallback(CVDisplayLinkRef displayLink, link->displayLinkEvent(inOutputTime); return kCVReturnSuccess; } +#endif AVFDisplayLink::AVFDisplayLink(QObject *parent) : QObject(parent) + , m_displayLink(0) , m_pendingDisplayLinkEvent(false) , m_isActive(false) { +#if defined(Q_OS_IOS) + m_displayLink = [[DisplayLinkObserver alloc] initWithAVFDisplayLink:this]; +#else // create display link for the main display CVDisplayLinkCreateWithCGDisplay(kCGDirectMainDisplay, &m_displayLink); if (m_displayLink) { @@ -80,6 +147,7 @@ AVFDisplayLink::AVFDisplayLink(QObject *parent) // set the renderer output callback function CVDisplayLinkSetOutputCallback(m_displayLink, &CVDisplayLinkCallback, this); } +#endif } AVFDisplayLink::~AVFDisplayLink() @@ -89,8 +157,12 @@ AVFDisplayLink::~AVFDisplayLink() #endif if (m_displayLink) { - CVDisplayLinkStop(m_displayLink); + stop(); +#if defined(Q_OS_IOS) + [_m_displayLink release]; +#else CVDisplayLinkRelease(m_displayLink); +#endif m_displayLink = NULL; } } @@ -108,20 +180,27 @@ bool AVFDisplayLink::isActive() const void AVFDisplayLink::start() { if (m_displayLink && !m_isActive) { - CVDisplayLinkStart(m_displayLink); - m_isActive = true; +#if defined(Q_OS_IOS) + [_m_displayLink start]; +#else + CVDisplayLinkStart(m_displayLink); +#endif + m_isActive = true; } } void AVFDisplayLink::stop() { if (m_displayLink && m_isActive) { +#if defined(Q_OS_IOS) + [_m_displayLink stop]; +#else CVDisplayLinkStop(m_displayLink); +#endif m_isActive = false; } } - void AVFDisplayLink::displayLinkEvent(const CVTimeStamp *ts) { // This function is called from a @@ -131,7 +210,12 @@ void AVFDisplayLink::displayLinkEvent(const CVTimeStamp *ts) m_displayLinkMutex.lock(); bool pending = m_pendingDisplayLinkEvent; m_pendingDisplayLinkEvent = true; +#if defined(Q_OS_IOS) + Q_UNUSED(ts); + memset(&m_frameTimeStamp, 0, sizeof(CVTimeStamp)); +#else m_frameTimeStamp = *ts; +#endif m_displayLinkMutex.unlock(); if (!pending) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.h b/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.h index 4cb58049e..69c233580 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.h +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.h @@ -57,6 +57,7 @@ private: AVFMediaPlayerControl *m_control; QMediaControl *m_videoOutput; AVFMediaPlayerMetaDataControl *m_playerMetaDataControl; + bool m_enableRenderControl; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.mm index bb75adb8b..34b42355c 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayerservice.mm @@ -43,26 +43,39 @@ #include "avfmediaplayersession.h" #include "avfmediaplayercontrol.h" #include "avfmediaplayermetadatacontrol.h" -#if defined(Q_OS_OSX) -# include "avfvideooutput.h" -# include "avfvideorenderercontrol.h" -#endif +#include "avfvideooutput.h" +#include "avfvideorenderercontrol.h" #ifndef QT_NO_WIDGETS # include "avfvideowidgetcontrol.h" #endif #include "avfvideowindowcontrol.h" +#if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_8, __IPHONE_6_0) +#import +#endif + QT_USE_NAMESPACE AVFMediaPlayerService::AVFMediaPlayerService(QObject *parent) : QMediaService(parent) , m_videoOutput(0) + , m_enableRenderControl(true) { m_session = new AVFMediaPlayerSession(this); m_control = new AVFMediaPlayerControl(this); m_control->setSession(m_session); m_playerMetaDataControl = new AVFMediaPlayerMetaDataControl(m_session, this); +#if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_8, __IPHONE_6_0) + // AVPlayerItemVideoOutput is available in SDK + #if QT_MAC_DEPLOYMENT_TARGET_BELOW(__MAC_10_8, __IPHONE_6_0) + // might not be available at runtime + #if defined(Q_OS_IOS) + m_enableRenderControl = [AVPlayerItemVideoOutput class] != 0; + #endif + #endif +#endif + connect(m_control, SIGNAL(mediaChanged(QMediaContent)), m_playerMetaDataControl, SLOT(updateTags())); } @@ -85,15 +98,16 @@ QMediaControl *AVFMediaPlayerService::requestControl(const char *name) if (qstrcmp(name, QMetaDataReaderControl_iid) == 0) return m_playerMetaDataControl; -#if defined(Q_OS_OSX) - if (qstrcmp(name, QVideoRendererControl_iid) == 0) { + + + if (m_enableRenderControl && (qstrcmp(name, QVideoRendererControl_iid) == 0)) { if (!m_videoOutput) m_videoOutput = new AVFVideoRendererControl(this); m_session->setVideoOutput(qobject_cast(m_videoOutput)); return m_videoOutput; } -#endif + #ifndef QT_NO_WIDGETS if (qstrcmp(name, QVideoWidgetControl_iid) == 0) { if (!m_videoOutput) @@ -119,11 +133,11 @@ void AVFMediaPlayerService::releaseControl(QMediaControl *control) qDebug() << Q_FUNC_INFO << control; #endif if (m_videoOutput == control) { -#if defined(Q_OS_OSX) AVFVideoRendererControl *renderControl = qobject_cast(m_videoOutput); + if (renderControl) renderControl->setSurface(0); -#endif + m_videoOutput = 0; m_session->setVideoOutput(0); diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h new file mode 100644 index 000000000..bcc8c64ac --- /dev/null +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h @@ -0,0 +1,107 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL21$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 or version 3 as published by the Free +** Software Foundation and appearing in the file LICENSE.LGPLv21 and +** LICENSE.LGPLv3 included in the packaging of this file. Please review the +** following information to ensure the GNU Lesser General Public License +** requirements will be met: https://www.gnu.org/licenses/lgpl.html and +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef AVFVIDEOFRAMERENDERER_H +#define AVFVIDEOFRAMERENDERER_H + +#include +#include +#include +#include + +@class AVPlayerLayer; +@class AVPlayerItemVideoOutput; + +QT_BEGIN_NAMESPACE + +class QOpenGLContext; +class QOpenGLFramebufferObject; +class QOpenGLShaderProgram; +class QOffscreenSurface; +class QAbstractVideoSurface; + +typedef struct __CVBuffer *CVBufferRef; +typedef CVBufferRef CVImageBufferRef; +typedef CVImageBufferRef CVPixelBufferRef; +#if defined(Q_OS_IOS) +typedef struct __CVOpenGLESTextureCache *CVOpenGLESTextureCacheRef; +typedef CVImageBufferRef CVOpenGLESTextureRef; +// helpers to avoid boring if def +typedef CVOpenGLESTextureCacheRef CVOGLTextureCacheRef; +typedef CVOpenGLESTextureRef CVOGLTextureRef; +#define CVOGLTextureGetTarget CVOpenGLESTextureGetTarget +#define CVOGLTextureGetName CVOpenGLESTextureGetName +#define CVOGLTextureCacheCreate CVOpenGLESTextureCacheCreate +#define CVOGLTextureCacheCreateTextureFromImage CVOpenGLESTextureCacheCreateTextureFromImage +#define CVOGLTextureCacheFlush CVOpenGLESTextureCacheFlush +#else +typedef struct __CVOpenGLTextureCache *CVOpenGLTextureCacheRef; +typedef CVImageBufferRef CVOpenGLTextureRef; +// helpers to avoid boring if def +typedef CVOpenGLTextureCacheRef CVOGLTextureCacheRef; +typedef CVOpenGLTextureRef CVOGLTextureRef; +#define CVOGLTextureGetTarget CVOpenGLTextureGetTarget +#define CVOGLTextureGetName CVOpenGLTextureGetName +#define CVOGLTextureCacheCreate CVOpenGLTextureCacheCreate +#define CVOGLTextureCacheCreateTextureFromImage CVOpenGLTextureCacheCreateTextureFromImage +#define CVOGLTextureCacheFlush CVOpenGLTextureCacheFlush +#endif + +class AVFVideoFrameRenderer : public QObject +{ +public: + AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent = 0); + + virtual ~AVFVideoFrameRenderer(); + + void setPlayerLayer(AVPlayerLayer *layer); + + CVOGLTextureRef renderLayerToTexture(AVPlayerLayer *layer); + QImage renderLayerToImage(AVPlayerLayer *layer); + +private: + void initRenderer(); + CVPixelBufferRef copyPixelBufferFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); + CVOGLTextureRef createCacheTextureFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); + + QOpenGLContext *m_glContext; + QOffscreenSurface *m_offscreenSurface; + QAbstractVideoSurface *m_surface; + CVOGLTextureCacheRef m_textureCache; + AVPlayerItemVideoOutput* m_videoOutput; + bool m_isContextShared; +}; + +QT_END_NAMESPACE + +#endif // AVFVIDEOFRAMERENDERER_H diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm new file mode 100644 index 000000000..31b9c38e1 --- /dev/null +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm @@ -0,0 +1,261 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "avfvideoframerenderer_ios.h" + +#include +#include +#include +#include + +#ifdef QT_DEBUG_AVF +#include +#endif + +#import +#import +QT_USE_NAMESPACE + +AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent) + : QObject(parent) + , m_glContext(0) + , m_offscreenSurface(0) + , m_surface(surface) + , m_textureCache(0) + , m_videoOutput(0) + , m_isContextShared(true) +{ +} + +AVFVideoFrameRenderer::~AVFVideoFrameRenderer() +{ +#ifdef QT_DEBUG_AVF + qDebug() << Q_FUNC_INFO; +#endif + + [m_videoOutput release]; // sending to nil is fine + if (m_textureCache) + CFRelease(m_textureCache); + delete m_offscreenSurface; + delete m_glContext; +} + +void AVFVideoFrameRenderer::setPlayerLayer(AVPlayerLayer *layer) +{ + Q_UNUSED(layer) + if (m_videoOutput) { + [m_videoOutput release]; + m_videoOutput = 0; + // will be re-created in first call to copyPixelBufferFromLayer + } +} + +CVOGLTextureRef AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +{ + initRenderer(); + + // If the glContext isn't shared, it doesn't make sense to return a texture for us + if (!m_isContextShared) + return 0; + + size_t dummyWidth = 0, dummyHeight = 0; + return createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); +} + +static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; +static NSNumber* const AVF_PIXEL_FORMAT_VALUE = [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]; +static NSDictionary* const AVF_OUTPUT_SETTINGS = [NSDictionary dictionaryWithObject:AVF_PIXEL_FORMAT_VALUE forKey:AVF_PIXEL_FORMAT_KEY]; + + +CVPixelBufferRef AVFVideoFrameRenderer::copyPixelBufferFromLayer(AVPlayerLayer *layer, + size_t& width, size_t& height) +{ + //Is layer valid + if (!layer) { +#ifdef QT_DEBUG_AVF + qWarning("copyPixelBufferFromLayer: invalid layer"); +#endif + return 0; + } + + if (!m_videoOutput) { + m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:AVF_OUTPUT_SETTINGS]; + [m_videoOutput setDelegate:nil queue:nil]; + AVPlayerItem * item = [[layer player] currentItem]; + [item addOutput:m_videoOutput]; + } + + CFTimeInterval currentCAFrameTime = CACurrentMediaTime(); + CMTime currentCMFrameTime = [m_videoOutput itemTimeForHostTime:currentCAFrameTime]; + // happens when buffering / loading + if (CMTimeCompare(currentCMFrameTime, kCMTimeZero) < 0) { + return 0; + } + + CVPixelBufferRef pixelBuffer = [m_videoOutput copyPixelBufferForItemTime:currentCMFrameTime + itemTimeForDisplay:nil]; + if (!pixelBuffer) { +#ifdef QT_DEBUG_AVF + qWarning("copyPixelBufferForItemTime returned nil"); + CMTimeShow(currentCMFrameTime); +#endif + return 0; + } + + width = CVPixelBufferGetWidth(pixelBuffer); + height = CVPixelBufferGetHeight(pixelBuffer); + return pixelBuffer; +} + +CVOGLTextureRef AVFVideoFrameRenderer::createCacheTextureFromLayer(AVPlayerLayer *layer, + size_t& width, size_t& height) +{ + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + + if (!pixelBuffer) + return 0; + + CVOGLTextureCacheFlush(m_textureCache, 0); + + CVOGLTextureRef texture = 0; + CVReturn err = CVOGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_textureCache, pixelBuffer, NULL, + GL_TEXTURE_2D, GL_RGBA, + (GLsizei) width, (GLsizei) height, + GL_BGRA, GL_UNSIGNED_BYTE, 0, + &texture); + + if (!texture || err) { +#ifdef QT_DEBUG_AVF + qWarning("CVOGLTextureCacheCreateTextureFromImage failed (error: %d)", err); +#endif + } + + CVPixelBufferRelease(pixelBuffer); + + return texture; +} + +QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) +{ + size_t width = 0; + size_t height = 0; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + + if (!pixelBuffer) + return QImage(); + + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixelFormat != kCVPixelFormatType_32BGRA) { +#ifdef QT_DEBUG_AVF + qWarning("CVPixelBuffer format is not BGRA32 (got: %d)", static_cast(pixelFormat)); +#endif + return QImage(); + } + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + char *data = (char *)CVPixelBufferGetBaseAddress(pixelBuffer); + size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer); + + // format here is not relevant, only using for storage + QImage img = QImage(width, height, QImage::Format_ARGB32); + for (size_t j = 0; j < height; j++) { + memcpy(img.scanLine(j), data, width * 4); + data += stride; + } + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + CVPixelBufferRelease(pixelBuffer); + return img; +} + +void AVFVideoFrameRenderer::initRenderer() +{ + // even for using a texture directly, we need to be able to make a context current, + // so we need an offscreen, and we shouldn't assume we can make the surface context + // current on that offscreen, so use our own (sharing with it). Slightly + // excessive but no performance penalty and makes the QImage path easier to maintain + + //Make sure we have an OpenGL context to make current + if (!m_glContext) { + //Create OpenGL context and set share context from surface + QOpenGLContext *shareContext = 0; + if (m_surface) { + shareContext = qobject_cast(m_surface->property("GLContext").value()); + } + + m_glContext = new QOpenGLContext(); + if (shareContext) { + m_glContext->setShareContext(shareContext); + m_isContextShared = true; + } else { +#ifdef QT_DEBUG_AVF + qWarning("failed to get Render Thread context"); +#endif + m_isContextShared = false; + } + if (!m_glContext->create()) { +#ifdef QT_DEBUG_AVF + qWarning("failed to create QOpenGLContext"); +#endif + return; + } + } + + if (!m_offscreenSurface) { + m_offscreenSurface = new QOffscreenSurface(); + m_offscreenSurface->setFormat(m_glContext->format()); + m_offscreenSurface->create(); + } + + //Need current context + m_glContext->makeCurrent(m_offscreenSurface); + + // Create a new open gl texture cache + CVReturn err = CVOGLTextureCacheCreate(kCFAllocatorDefault, NULL, + [EAGLContext currentContext], + NULL, &m_textureCache); + + if (err) { +#ifdef QT_DEBUG_AVF + qWarning("Error at CVOGLTextureCacheCreate %d", err); +#endif + } +} diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index c7087b612..3ca9c1873 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -41,26 +41,40 @@ #include "avfvideorenderercontrol.h" #include "avfdisplaylink.h" + +#if defined(Q_OS_IOS) +#include "avfvideoframerenderer_ios.h" +#else #include "avfvideoframerenderer.h" +#endif #include #include #include + +#include + #include #import QT_USE_NAMESPACE -class TextureVideoBuffer : public QAbstractVideoBuffer +#if defined(Q_OS_IOS) +class TextureCacheVideoBuffer : public QAbstractVideoBuffer { public: - TextureVideoBuffer(GLuint textureId) + TextureCacheVideoBuffer(CVOGLTextureRef texture) : QAbstractVideoBuffer(GLTextureHandle) - , m_textureId(textureId) + , m_texture(texture) {} - virtual ~TextureVideoBuffer() {} + virtual ~TextureCacheVideoBuffer() + { + // absolutely critical that we drop this + // reference of textures will stay in the cache + CFRelease(m_texture); + } MapMode mapMode() const { return NotMapped; } uchar *map(MapMode, int*, int*) { return 0; } @@ -68,41 +82,39 @@ public: QVariant handle() const { - return QVariant::fromValue(m_textureId); + GLuint texId = CVOGLTextureGetName(m_texture); + return QVariant::fromValue(texId); } private: - GLuint m_textureId; + CVOGLTextureRef m_texture; }; - -class QImageVideoBuffer : public QAbstractVideoBuffer +#else +class TextureVideoBuffer : public QAbstractVideoBuffer { public: - QImageVideoBuffer(const QImage &image) - : QAbstractVideoBuffer(NoHandle) - , m_image(image) - , m_mode(NotMapped) - { + TextureVideoBuffer(GLuint tex) + : QAbstractVideoBuffer(GLTextureHandle) + , m_texture(tex) + {} + virtual ~TextureVideoBuffer() + { } - MapMode mapMode() const { return m_mode; } - uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) + MapMode mapMode() const { return NotMapped; } + uchar *map(MapMode, int*, int*) { return 0; } + void unmap() {} + + QVariant handle() const { - if (mode != NotMapped && m_mode == NotMapped) { - m_mode = mode; - return m_image.bits(); - } else - return 0; + return QVariant::fromValue(m_texture); } - void unmap() { - m_mode = NotMapped; - } private: - QImage m_image; - MapMode m_mode; + GLuint m_texture; }; +#endif AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) @@ -122,8 +134,7 @@ AVFVideoRendererControl::~AVFVideoRendererControl() qDebug() << Q_FUNC_INFO; #endif m_displayLink->stop(); - if (m_playerLayer) - [(AVPlayerLayer*)m_playerLayer release]; + [(AVPlayerLayer*)m_playerLayer release]; } QAbstractVideoSurface *AVFVideoRendererControl::surface() const @@ -150,8 +161,8 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) m_surface = surface; //If the surface changed, then the current frame renderer is no longer valid - if (m_frameRenderer) - delete m_frameRenderer; + delete m_frameRenderer; + m_frameRenderer = 0; //If there is now no surface to render too if (m_surface == 0) { @@ -161,6 +172,11 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) //Surface changed, so we need a new frame renderer m_frameRenderer = new AVFVideoFrameRenderer(m_surface, this); +#if defined(Q_OS_IOS) + if (m_playerLayer) { + m_frameRenderer->setPlayerLayer(static_cast(m_playerLayer)); + } +#endif //Check for needed formats to render as OpenGL Texture m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); @@ -187,6 +203,12 @@ void AVFVideoRendererControl::setLayer(void *playerLayer) if (m_surface && m_surface->isActive()) m_surface->stop(); +#if defined(Q_OS_IOS) + if (m_frameRenderer) { + m_frameRenderer->setPlayerLayer(static_cast(playerLayer)); + } +#endif + //If there is no layer to render, stop scheduling updates if (m_playerLayer == 0) { m_displayLink->stop(); @@ -216,16 +238,22 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) return; if (m_enableOpenGL) { +#if defined(Q_OS_IOS) + CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer); - GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer); + //Make sure we got a valid texture + if (tex == 0) + return; + QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex); +#else + GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer); //Make sure we got a valid texture - if (textureId == 0) { - qWarning("renderLayerToTexture failed"); + if (tex == 0) return; - } - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(textureId); + QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex); +#endif QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); if (m_surface && frame.isValid()) { @@ -234,8 +262,11 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) if (!m_surface->isActive()) { QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::GLTextureHandle); +#if defined(Q_OS_IOS) + format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); +#else format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); - +#endif if (!m_surface->start(format)) { //Surface doesn't support GLTextureHandle qWarning("Failed to activate video surface"); @@ -250,13 +281,11 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer); if (frameData.isNull()) { - qWarning("renterLayerToImage failed"); return; } QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData); - QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32_Premultiplied); - + QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32); if (m_surface && frame.isValid()) { if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) m_surface->stop(); diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index 2670eceaf..28cdc2727 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -44,21 +44,31 @@ OBJECTIVE_SOURCES += \ avfvideowidget.mm } -!ios { - LIBS += -framework QuartzCore -framework AppKit +ios { + contains(QT_CONFIG, opengl.*) { + HEADERS += \ + avfvideoframerenderer_ios.h \ + avfvideorenderercontrol.h \ + avfdisplaylink.h - HEADERS += \ - avfvideorenderercontrol.h \ - avfdisplaylink.h - OBJECTIVE_SOURCES += \ - avfvideorenderercontrol.mm \ - avfdisplaylink.mm + OBJECTIVE_SOURCES += \ + avfvideoframerenderer_ios.mm \ + avfvideorenderercontrol.mm \ + avfdisplaylink.mm + } +} else { + LIBS += -framework QuartzCore -framework AppKit contains(QT_CONFIG, opengl.*) { HEADERS += \ - avfvideoframerenderer.h + avfvideoframerenderer.h \ + avfvideorenderercontrol.h \ + avfdisplaylink.h + OBJECTIVE_SOURCES += \ - avfvideoframerenderer.mm + avfvideoframerenderer.mm \ + avfvideorenderercontrol.mm \ + avfdisplaylink.mm } } -- cgit v1.2.3