diff options
Diffstat (limited to 'src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm')
| -rw-r--r-- | src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm | 464 |
1 files changed, 0 insertions, 464 deletions
diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm deleted file mode 100644 index f57b26dff..000000000 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ /dev/null @@ -1,464 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "avfvideoframerenderer.h" - -#include <QtMultimedia/qabstractvideosurface.h> -#include <QtOpenGL/QOpenGLFramebufferObject> -#include <QtGui/QWindow> -#include <QOpenGLShaderProgram> - -#ifdef QT_DEBUG_AVF -#include <QtCore/qdebug.h> -#endif - -#import <CoreVideo/CVBase.h> -#import <AVFoundation/AVFoundation.h> - -QT_USE_NAMESPACE - -AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent) - : QObject(parent) - , m_videoLayerRenderer(nullptr) - , m_surface(surface) - , m_offscreenSurface(nullptr) - , m_glContext(nullptr) - , m_currentBuffer(1) - , m_isContextShared(true) -{ - m_fbo[0] = nullptr; - m_fbo[1] = nullptr; -} - -AVFVideoFrameRenderer::~AVFVideoFrameRenderer() -{ -#ifdef QT_DEBUG_AVF - qDebug() << Q_FUNC_INFO; -#endif - - [m_videoLayerRenderer release]; - [m_metalDevice release]; - delete m_fbo[0]; - delete m_fbo[1]; - delete m_offscreenSurface; - delete m_glContext; - - if (m_useCoreProfile) { - glDeleteVertexArrays(1, &m_quadVao); - glDeleteBuffers(2, m_quadVbos); - delete m_shader; - } -} - -quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) -{ - //Is layer valid - if (!layer) - return 0; - - //If the glContext isn't shared, it doesn't make sense to return a texture for us - if (m_offscreenSurface && !m_isContextShared) - return 0; - - QOpenGLFramebufferObject *fbo = initRenderer(layer); - - if (!fbo) - return 0; - - renderLayerToFBO(layer, fbo); - if (m_glContext) - m_glContext->doneCurrent(); - - return fbo->texture(); -} - -quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) -{ - m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); - - if (!m_metalDevice) - m_metalDevice = MTLCreateSystemDefaultDevice(); - - if (!m_metalTexture) { - auto desc = [[MTLTextureDescriptor alloc] init]; - desc.textureType = MTLTextureType2D; - desc.width = NSUInteger(m_targetSize.width()); - desc.height = NSUInteger(m_targetSize.height()); - desc.resourceOptions = MTLResourceStorageModePrivate; - desc.usage = MTLTextureUsageRenderTarget; - desc.pixelFormat = MTLPixelFormatRGBA8Unorm; - - m_metalTexture = [m_metalDevice newTextureWithDescriptor: desc]; - [desc release]; - } - - if (!m_videoLayerRenderer) { - m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; - [m_videoLayerRenderer retain]; - } - - if (m_videoLayerRenderer.layer != layer) { - m_videoLayerRenderer.layer = layer; - m_videoLayerRenderer.bounds = layer.bounds; - } - - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; - [m_videoLayerRenderer addUpdateRect:layer.bounds]; - [m_videoLayerRenderer render]; - [m_videoLayerRenderer endFrame]; - - return quint64(m_metalTexture); -} - -QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) -{ - //Is layer valid - if (!layer) { - return QImage(); - } - - QOpenGLFramebufferObject *fbo = initRenderer(layer); - - if (!fbo) - return QImage(); - - renderLayerToFBO(layer, fbo); - QImage fboImage = fbo->toImage(); - if (m_glContext) - m_glContext->doneCurrent(); - - return fboImage; -} - -QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *layer) -{ - - //Get size from AVPlayerLayer - m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); - - QOpenGLContext *shareContext = !m_glContext && m_surface - ? qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>()) - : nullptr; - - //Make sure we have an OpenGL context to make current - if (shareContext || (!QOpenGLContext::currentContext() && !m_glContext)) { - - //Create Hidden QWindow surface to create context in this thread - delete m_offscreenSurface; - m_offscreenSurface = new QWindow(); - m_offscreenSurface->setSurfaceType(QWindow::OpenGLSurface); - //Needs geometry to be a valid surface, but size is not important - m_offscreenSurface->setGeometry(0, 0, 1, 1); - m_offscreenSurface->create(); - - delete m_glContext; - m_glContext = new QOpenGLContext(); - m_glContext->setFormat(m_offscreenSurface->requestedFormat()); - - if (shareContext) { - m_glContext->setShareContext(shareContext); - m_isContextShared = true; - } else { -#ifdef QT_DEBUG_AVF - qWarning("failed to get Render Thread context"); -#endif - m_isContextShared = false; - } - if (!m_glContext->create()) { - qWarning("failed to create QOpenGLContext"); - return nullptr; - } - - // CARenderer must be re-created with different current context, so release it now. - // See lines below where m_videoLayerRenderer is constructed. - if (m_videoLayerRenderer) { - [m_videoLayerRenderer release]; - m_videoLayerRenderer = nullptr; - } - - if (m_useCoreProfile) { - glDeleteVertexArrays(1, &m_quadVao); - glDeleteBuffers(2, m_quadVbos); - delete m_shader; - m_shader = nullptr; - } - } - - //Need current context - if (m_glContext) - m_glContext->makeCurrent(m_offscreenSurface); - - if (!m_metalDevice) - m_metalDevice = MTLCreateSystemDefaultDevice(); - - if (@available(macOS 10.13, *)) { - m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() == - QSurfaceFormat::CoreProfile); - } else { - m_useCoreProfile = false; - } - - // Create the CARenderer if needed for no Core OpenGL - if (!m_videoLayerRenderer) { - if (!m_useCoreProfile) { - m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() - options: nil]; - [m_videoLayerRenderer retain]; - } else if (@available(macOS 10.13, *)) { - // This is always true when m_useCoreProfile is true, but the compiler wants the check - // anyway - // Setup Core OpenGL shader, VAO, VBOs and metal renderer - m_shader = new QOpenGLShaderProgram(); - m_shader->create(); - if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core - in vec2 qt_VertexPosition; - in vec2 qt_VertexTexCoord; - out vec2 qt_TexCoord; - void main() - { - qt_TexCoord = qt_VertexTexCoord; - gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f); - })")) { - qCritical() << "Vertex shader compilation failed" << m_shader->log(); - } - if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core - in vec2 qt_TexCoord; - out vec4 fragColor; - uniform sampler2DRect videoFrame; - void main(void) - { - ivec2 textureDim = textureSize(videoFrame); - fragColor = texture(videoFrame, qt_TexCoord * textureDim); - })")) { - qCritical() << "Fragment shader compilation failed" << m_shader->log(); - } - - // Setup quad where the video frame will be attached - GLfloat vertices[] = { - -1.0f, -1.0f, - 1.0f, -1.0f, - -1.0f, 1.0f, - 1.0f, 1.0f, - }; - - GLfloat uvs[] = { - 0.0f, 0.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, - }; - - glGenVertexArrays(1, &m_quadVao); - glBindVertexArray(m_quadVao); - - // Create vertex buffer objects for vertices - glGenBuffers(2, m_quadVbos); - - // Setup vertices - glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]); - glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); - glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); - glEnableVertexAttribArray(0); - - // Setup uvs - glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]); - glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW); - glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); - glEnableVertexAttribArray(1); - - glBindVertexArray(0); - - // Setup shared Metal/OpenGL pixel buffer and textures - m_NSGLContext = QOpenGLContext::currentContext()->nativeInterface<QNativeInterface::QCocoaGLContext>()->nativeContext(); - m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj; - - NSDictionary* cvBufferProperties = @{ - static_cast<NSString*>(kCVPixelBufferOpenGLCompatibilityKey) : @YES, - static_cast<NSString*>(kCVPixelBufferMetalCompatibilityKey): @YES, - }; - - CVPixelBufferCreate(kCFAllocatorDefault, static_cast<size_t>(m_targetSize.width()), - static_cast<size_t>(m_targetSize.height()), kCVPixelFormatType_32BGRA, - static_cast<CFDictionaryRef>(cvBufferProperties), &m_CVPixelBuffer); - - m_textureName = createGLTexture(reinterpret_cast<CGLContextObj>(m_NSGLContext.CGLContextObj), - m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer, - m_CVGLTexture); - m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer, - MTLPixelFormatBGRA8Unorm, - static_cast<size_t>(m_targetSize.width()), - static_cast<size_t>(m_targetSize.height()), - m_CVMTLTexture); - - m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; - [m_videoLayerRenderer retain]; - } - } - - //Set/Change render source if needed - if (m_videoLayerRenderer.layer != layer) { - m_videoLayerRenderer.layer = layer; - m_videoLayerRenderer.bounds = layer.bounds; - } - - //Do we have FBO's already? - if ((!m_fbo[0] && !m_fbo[0]) || (m_fbo[0]->size() != m_targetSize)) { - delete m_fbo[0]; - delete m_fbo[1]; - m_fbo[0] = new QOpenGLFramebufferObject(m_targetSize); - m_fbo[1] = new QOpenGLFramebufferObject(m_targetSize); - } - - //Switch buffer target - m_currentBuffer = !m_currentBuffer; - return m_fbo[m_currentBuffer]; -} - -void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo) -{ - //Start Rendering - //NOTE: This rendering method will NOT work on iOS as there is no CARenderer in iOS - if (!fbo->bind()) { - qWarning("AVFVideoRender FBO failed to bind"); - return; - } - - glClearColor(0.0f, 0.0f, 0.0f, 1.0f); - glClear(GL_COLOR_BUFFER_BIT); - - glViewport(0, 0, m_targetSize.width(), m_targetSize.height()); - - if (m_useCoreProfile) { - CGLLockContext(m_NSGLContext.CGLContextObj); - m_shader->bind(); - glBindVertexArray(m_quadVao); - } else { - glMatrixMode(GL_PROJECTION); - glPushMatrix(); - glLoadIdentity(); - - // Render to FBO with inverted Y - glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); - - glMatrixMode(GL_MODELVIEW); - glPushMatrix(); - glLoadIdentity(); - } - - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr]; - [m_videoLayerRenderer addUpdateRect:layer.bounds]; - [m_videoLayerRenderer render]; - [m_videoLayerRenderer endFrame]; - - if (m_useCoreProfile) { - glActiveTexture(0); - glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName); - - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - glBindTexture(GL_TEXTURE_RECTANGLE, 0); - - glBindVertexArray(0); - - m_shader->release(); - - CGLFlushDrawable(m_NSGLContext.CGLContextObj); - CGLUnlockContext(m_NSGLContext.CGLContextObj); - } else { - glMatrixMode(GL_MODELVIEW); - glPopMatrix(); - glMatrixMode(GL_PROJECTION); - glPopMatrix(); - } - - glFinish(); //Rendering needs to be done before passing texture to video frame - - fbo->release(); -} - -GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache, - CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef) -{ - CVReturn cvret; - // Create an OpenGL CoreVideo texture cache from the pixel buffer. - cvret = CVOpenGLTextureCacheCreate( - kCFAllocatorDefault, - nil, - cglContextObj, - cglPixelFormtObj, - nil, - &cvglTextureCache); - - // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache. - cvret = CVOpenGLTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - cvglTextureCache, - cvPixelBufferRef, - nil, - &cvOpenGLTextureRef); - - // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image. - return CVOpenGLTextureGetName(cvOpenGLTextureRef); -} - -id<MTLTexture> AVFVideoFrameRenderer::createMetalTexture(id<MTLDevice> mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef, - MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef) -{ - CVReturn cvret; - // Create a Metal Core Video texture cache from the pixel buffer. - cvret = CVMetalTextureCacheCreate( - kCFAllocatorDefault, - nil, - mtlDevice, - nil, - &cvMetalTextureCacheRef); - - // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. - cvret = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - cvMetalTextureCacheRef, - cvPixelBufferRef, nil, - pixelFormat, - width, height, - 0, - &cvMetalTextureRef); - - // Get a Metal texture using the CoreVideo Metal texture reference. - return CVMetalTextureGetTexture(cvMetalTextureRef); -} |