iOS video frame render implementation.

Uses CVTextureCache, iOS only for now, OS-X code could be ported
but will need further work to support TEXTURE_RECTANGLE in the
QVideoNode classes.

When we can’t share a context, falls back to an offscreen window,
FBO rendering and grabbing a QImage.

Change-Id: I23b831fdcc63aeb1b67b7741d8d56779470240d3
Reviewed-by: Yoann Lopes <yoann.lopes@theqtcompany.com>

1 files changed, 67 insertions, 38 deletions

diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm
index c7087b612..3ca9c1873 100644
--- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm
+++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm
@@ -41,26 +41,40 @@
 
 #include "avfvideorenderercontrol.h"
 #include "avfdisplaylink.h"
+
+#if defined(Q_OS_IOS)
+#include "avfvideoframerenderer_ios.h"
+#else
 #include "avfvideoframerenderer.h"
+#endif
 
 #include <QtMultimedia/qabstractvideobuffer.h>
 #include <QtMultimedia/qabstractvideosurface.h>
 #include <QtMultimedia/qvideosurfaceformat.h>
+
+#include <private/qimagevideobuffer_p.h>
+
 #include <QtCore/qdebug.h>
 
 #import <AVFoundation/AVFoundation.h>
 
 QT_USE_NAMESPACE
 
-class TextureVideoBuffer : public QAbstractVideoBuffer
+#if defined(Q_OS_IOS)
+class TextureCacheVideoBuffer : public QAbstractVideoBuffer
 {
 public:
-    TextureVideoBuffer(GLuint textureId)
+    TextureCacheVideoBuffer(CVOGLTextureRef texture)
         : QAbstractVideoBuffer(GLTextureHandle)
-        , m_textureId(textureId)
+        , m_texture(texture)
     {}
 
-    virtual ~TextureVideoBuffer() {}
+    virtual ~TextureCacheVideoBuffer()
+    {
+        // absolutely critical that we drop this
+        // reference of textures will stay in the cache
+        CFRelease(m_texture);
+    }
 
     MapMode mapMode() const { return NotMapped; }
     uchar *map(MapMode, int*, int*) { return 0; }
@@ -68,41 +82,39 @@ public:
 
     QVariant handle() const
     {
-        return QVariant::fromValue<unsigned int>(m_textureId);
+        GLuint texId = CVOGLTextureGetName(m_texture);
+        return QVariant::fromValue<unsigned int>(texId);
     }
 
 private:
-    GLuint m_textureId;
+    CVOGLTextureRef m_texture;
 };
-
-class QImageVideoBuffer : public QAbstractVideoBuffer
+#else
+class TextureVideoBuffer : public QAbstractVideoBuffer
 {
 public:
-    QImageVideoBuffer(const QImage &image)
-        : QAbstractVideoBuffer(NoHandle)
-        , m_image(image)
-        , m_mode(NotMapped)
-    {
+    TextureVideoBuffer(GLuint  tex)
+        : QAbstractVideoBuffer(GLTextureHandle)
+        , m_texture(tex)
+    {}
 
+    virtual ~TextureVideoBuffer()
+    {
     }
 
-    MapMode mapMode() const { return m_mode; }
-    uchar *map(MapMode mode, int *numBytes, int *bytesPerLine)
+    MapMode mapMode() const { return NotMapped; }
+    uchar *map(MapMode, int*, int*) { return 0; }
+    void unmap() {}
+
+    QVariant handle() const
     {
-        if (mode != NotMapped && m_mode == NotMapped) {
-            m_mode = mode;
-            return m_image.bits();
-        } else
-            return 0;
+        return QVariant::fromValue<unsigned int>(m_texture);
     }
 
-    void unmap() {
-        m_mode = NotMapped;
-    }
 private:
-    QImage m_image;
-    MapMode m_mode;
+    GLuint m_texture;
 };
+#endif
 
 AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent)
     : QVideoRendererControl(parent)
@@ -122,8 +134,7 @@ AVFVideoRendererControl::~AVFVideoRendererControl()
     qDebug() << Q_FUNC_INFO;
 #endif
     m_displayLink->stop();
-    if (m_playerLayer)
-        [(AVPlayerLayer*)m_playerLayer release];
+    [(AVPlayerLayer*)m_playerLayer release];
 }
 
 QAbstractVideoSurface *AVFVideoRendererControl::surface() const
@@ -150,8 +161,8 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface)
     m_surface = surface;
 
     //If the surface changed, then the current frame renderer is no longer valid
-    if (m_frameRenderer)
-        delete m_frameRenderer;
+    delete m_frameRenderer;
+    m_frameRenderer = 0;
 
     //If there is now no surface to render too
     if (m_surface == 0) {
@@ -161,6 +172,11 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface)
 
     //Surface changed, so we need a new frame renderer
     m_frameRenderer = new AVFVideoFrameRenderer(m_surface, this);
+#if defined(Q_OS_IOS)
+    if (m_playerLayer) {
+        m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(m_playerLayer));
+    }
+#endif
 
     //Check for needed formats to render as OpenGL Texture
     m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32);
@@ -187,6 +203,12 @@ void AVFVideoRendererControl::setLayer(void *playerLayer)
     if (m_surface && m_surface->isActive())
         m_surface->stop();
 
+#if defined(Q_OS_IOS)
+    if (m_frameRenderer) {
+        m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(playerLayer));
+    }
+#endif
+
     //If there is no layer to render, stop scheduling updates
     if (m_playerLayer == 0) {
         m_displayLink->stop();
@@ -216,16 +238,22 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
         return;
 
     if (m_enableOpenGL) {
+#if defined(Q_OS_IOS)
+        CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer);
 
-        GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer);
+        //Make sure we got a valid texture
+        if (tex == 0)
+            return;
 
+        QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex);
+#else
+        GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer);
         //Make sure we got a valid texture
-        if (textureId == 0) {
-            qWarning("renderLayerToTexture failed");
+        if (tex == 0)
             return;
-        }
 
-        QAbstractVideoBuffer *buffer = new TextureVideoBuffer(textureId);
+        QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex);
+#endif
         QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32);
 
         if (m_surface && frame.isValid()) {
@@ -234,8 +262,11 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
 
             if (!m_surface->isActive()) {
                 QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::GLTextureHandle);
+#if defined(Q_OS_IOS)
+                format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom);
+#else
                 format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop);
-
+#endif
                 if (!m_surface->start(format)) {
                     //Surface doesn't support GLTextureHandle
                     qWarning("Failed to activate video surface");
@@ -250,13 +281,11 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
         QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer);
 
         if (frameData.isNull()) {
-            qWarning("renterLayerToImage failed");
             return;
         }
 
         QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData);
-        QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32_Premultiplied);
-
+        QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32);
         if (m_surface && frame.isValid()) {
             if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat())
                 m_surface->stop();