diff options
author | Tarja Sundqvist <tarja.sundqvist@qt.io> | 2022-04-07 15:22:10 +0300 |
---|---|---|
committer | Tarja Sundqvist <tarja.sundqvist@qt.io> | 2022-04-07 15:22:10 +0300 |
commit | dc0ee181d4c56ff66c989bd3292dad7cc35e234b (patch) | |
tree | e8934d4e5e93d60ed957afd0d2b1a6d5319c14ed | |
parent | fa6c3d653682f9fd331d859c7196a291a8a4d8d5 (diff) | |
parent | 51282e083d7a1c06309cf7f2fc216f6203b4fbb2 (diff) |
Merge remote-tracking branch 'origin/tqtc/lts-5.15.4' into tqtc/lts-5.15-opensourcev5.15.4-lts-lgpl
Change-Id: I6568139d7193f5b04b50f0f31ddff4d9bac8e65a
-rw-r--r-- | .qmake.conf | 2 | ||||
-rw-r--r-- | src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm | 3 | ||||
-rw-r--r-- | src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h | 39 | ||||
-rw-r--r-- | src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm | 225 | ||||
-rw-r--r-- | src/plugins/avfoundation/mediaplayer/mediaplayer.pro | 3 | ||||
-rw-r--r-- | tests/auto/unit/qvideoframe/tst_qvideoframe.cpp | 8 |
6 files changed, 255 insertions, 25 deletions
diff --git a/.qmake.conf b/.qmake.conf index 5fb75cffe..163ffef25 100644 --- a/.qmake.conf +++ b/.qmake.conf @@ -2,4 +2,4 @@ load(qt_build_config) DEFINES += QT_NO_FOREACH QT_NO_JAVA_STYLE_ITERATORS QT_NO_LINKED_LIST -MODULE_VERSION = 5.15.3 +MODULE_VERSION = 5.15.4 diff --git a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm index 9e1bf3f84..7bf9de071 100644 --- a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm +++ b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm @@ -278,7 +278,8 @@ AVFCameraRendererControl::AVFCameraRendererControl(QObject *parent) AVFCameraRendererControl::~AVFCameraRendererControl() { - [m_cameraSession->captureSession() removeOutput:m_videoDataOutput]; + if ([m_cameraSession->captureSession().outputs containsObject:m_videoDataOutput]) + [m_cameraSession->captureSession() removeOutput:m_videoDataOutput]; [m_viewfinderFramesDelegate release]; if (m_delegateQueue) dispatch_release(m_delegateQueue); diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index 99b6bb0b5..c5d7fe66c 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -45,12 +45,16 @@ #include <QtGui/QOpenGLContext> #include <QtCore/QSize> +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class CARenderer; @class AVPlayerLayer; QT_BEGIN_NAMESPACE class QOpenGLFramebufferObject; +class QOpenGLShaderProgram; class QWindow; class QOpenGLContext; class QAbstractVideoSurface; @@ -65,17 +69,52 @@ public: GLuint renderLayerToTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); + static GLuint createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, + CVOpenGLTextureCacheRef cvglTextureCache, + CVPixelBufferRef cvPixelBufferRef, + CVOpenGLTextureRef cvOpenGLTextureRef); + + static id<MTLTexture> createMetalTexture(id<MTLDevice> mtlDevice, + CVMetalTextureCacheRef cvMetalTextureCacheRef, + CVPixelBufferRef cvPixelBufferRef, + MTLPixelFormat pixelFormat, size_t width, size_t height, + CVMetalTextureRef cvMetalTextureRef); + private: QOpenGLFramebufferObject* initRenderer(AVPlayerLayer *layer); void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); + void renderLayerToFBOCoreOpenGL(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); CARenderer *m_videoLayerRenderer; QAbstractVideoSurface *m_surface; QOpenGLFramebufferObject *m_fbo[2]; + QOpenGLShaderProgram *m_shader = nullptr; QWindow *m_offscreenSurface; QOpenGLContext *m_glContext; QSize m_targetSize; + bool m_useCoreProfile = false; + + // Shared pixel buffer + CVPixelBufferRef m_CVPixelBuffer; + + // OpenGL Texture + CVOpenGLTextureCacheRef m_CVGLTextureCache; + CVOpenGLTextureRef m_CVGLTexture; + CGLPixelFormatObj m_CGLPixelFormat; + GLuint m_textureName = 0; + + // Metal Texture + CVMetalTextureRef m_CVMTLTexture; + CVMetalTextureCacheRef m_CVMTLTextureCache; + id<MTLDevice> m_metalDevice = nil; + id<MTLTexture> m_metalTexture = nil; + + NSOpenGLContext *m_NSGLContext = nullptr; + + GLuint m_quadVao = 0; + GLuint m_quadVbos[2]; + uint m_currentBuffer; bool m_isContextShared; }; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 51f961729..23e5497a2 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -42,6 +42,8 @@ #include <QtMultimedia/qabstractvideosurface.h> #include <QtGui/QOpenGLFramebufferObject> #include <QtGui/QWindow> +#include <QOpenGLShaderProgram> +#include <QtPlatformHeaders/QCocoaNativeContext> #ifdef QT_DEBUG_AVF #include <QtCore/qdebug.h> @@ -76,6 +78,12 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() delete m_fbo[1]; delete m_offscreenSurface; delete m_glContext; + + if (m_useCoreProfile) { + glDeleteVertexArrays(1, &m_quadVao); + glDeleteBuffers(2, m_quadVbos); + delete m_shader; + } } GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) @@ -166,16 +174,124 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay [m_videoLayerRenderer release]; m_videoLayerRenderer = nullptr; } + + if (m_useCoreProfile) { + glDeleteVertexArrays(1, &m_quadVao); + glDeleteBuffers(2, m_quadVbos); + delete m_shader; + m_shader = nullptr; + } } //Need current context if (m_glContext) m_glContext->makeCurrent(m_offscreenSurface); - //Create the CARenderer if needed + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (@available(macOS 10.13, *)) { + m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() == + QSurfaceFormat::CoreProfile); + } else { + m_useCoreProfile = false; + } + + // Create the CARenderer if needed for no Core OpenGL if (!m_videoLayerRenderer) { - m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() options: nil]; - [m_videoLayerRenderer retain]; + if (!m_useCoreProfile) { + m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() + options: nil]; + [m_videoLayerRenderer retain]; + } else if (@available(macOS 10.13, *)) { + // This is always true when m_useCoreProfile is true, but the compiler wants the check + // anyway + // Setup Core OpenGL shader, VAO, VBOs and metal renderer + m_shader = new QOpenGLShaderProgram(); + m_shader->create(); + if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core + in vec2 qt_VertexPosition; + in vec2 qt_VertexTexCoord; + out vec2 qt_TexCoord; + void main() + { + qt_TexCoord = qt_VertexTexCoord; + gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f); + })")) { + qCritical() << "Vertex shader compilation failed" << m_shader->log(); + } + if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core + in vec2 qt_TexCoord; + out vec4 fragColor; + uniform sampler2DRect videoFrame; + void main(void) + { + ivec2 textureDim = textureSize(videoFrame); + fragColor = texture(videoFrame, qt_TexCoord * textureDim); + })")) { + qCritical() << "Fragment shader compilation failed" << m_shader->log(); + } + + // Setup quad where the video frame will be attached + GLfloat vertices[] = { + -1.0f, -1.0f, + 1.0f, -1.0f, + -1.0f, 1.0f, + 1.0f, 1.0f, + }; + + GLfloat uvs[] = { + 0.0f, 0.0f, + 1.0f, 0.0f, + 0.0f, 1.0f, + 1.0f, 1.0f, + }; + + glGenVertexArrays(1, &m_quadVao); + glBindVertexArray(m_quadVao); + + // Create vertex buffer objects for vertices + glGenBuffers(2, m_quadVbos); + + // Setup vertices + glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]); + glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); + glEnableVertexAttribArray(0); + + // Setup uvs + glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]); + glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW); + glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); + glEnableVertexAttribArray(1); + + glBindVertexArray(0); + + // Setup shared Metal/OpenGL pixel buffer and textures + m_NSGLContext = static_cast<QCocoaNativeContext*>((QOpenGLContext::currentContext()->nativeHandle().data()))->context(); + m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj; + + NSDictionary* cvBufferProperties = @{ + static_cast<NSString*>(kCVPixelBufferOpenGLCompatibilityKey) : @YES, + static_cast<NSString*>(kCVPixelBufferMetalCompatibilityKey): @YES, + }; + + CVPixelBufferCreate(kCFAllocatorDefault, static_cast<size_t>(m_targetSize.width()), + static_cast<size_t>(m_targetSize.height()), kCVPixelFormatType_32BGRA, + static_cast<CFDictionaryRef>(cvBufferProperties), &m_CVPixelBuffer); + + m_textureName = createGLTexture(reinterpret_cast<CGLContextObj>(m_NSGLContext.CGLContextObj), + m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer, + m_CVGLTexture); + m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer, + MTLPixelFormatBGRA8Unorm, + static_cast<size_t>(m_targetSize.width()), + static_cast<size_t>(m_targetSize.height()), + m_CVMTLTexture); + + m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; + [m_videoLayerRenderer retain]; + } } //Set/Change render source if needed @@ -211,28 +327,101 @@ void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFrameb glViewport(0, 0, m_targetSize.width(), m_targetSize.height()); - glMatrixMode(GL_PROJECTION); - glPushMatrix(); - glLoadIdentity(); - - //Render to FBO with inverted Y - glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); - - glMatrixMode(GL_MODELVIEW); - glPushMatrix(); - glLoadIdentity(); + if (m_useCoreProfile) { + CGLLockContext(m_NSGLContext.CGLContextObj); + m_shader->bind(); + glBindVertexArray(m_quadVao); + } else { + glMatrixMode(GL_PROJECTION); + glPushMatrix(); + glLoadIdentity(); + + // Render to FBO with inverted Y + glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); + + glMatrixMode(GL_MODELVIEW); + glPushMatrix(); + glLoadIdentity(); + } - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr]; [m_videoLayerRenderer addUpdateRect:layer.bounds]; [m_videoLayerRenderer render]; [m_videoLayerRenderer endFrame]; - glMatrixMode(GL_MODELVIEW); - glPopMatrix(); - glMatrixMode(GL_PROJECTION); - glPopMatrix(); + if (m_useCoreProfile) { + glActiveTexture(0); + glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName); + + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + + glBindTexture(GL_TEXTURE_RECTANGLE, 0); + + glBindVertexArray(0); + + m_shader->release(); + + CGLFlushDrawable(m_NSGLContext.CGLContextObj); + CGLUnlockContext(m_NSGLContext.CGLContextObj); + } else { + glMatrixMode(GL_MODELVIEW); + glPopMatrix(); + glMatrixMode(GL_PROJECTION); + glPopMatrix(); + } glFinish(); //Rendering needs to be done before passing texture to video frame fbo->release(); } + +GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache, + CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef) +{ + CVReturn cvret; + // Create an OpenGL CoreVideo texture cache from the pixel buffer. + cvret = CVOpenGLTextureCacheCreate( + kCFAllocatorDefault, + nil, + cglContextObj, + cglPixelFormtObj, + nil, + &cvglTextureCache); + + // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache. + cvret = CVOpenGLTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + cvglTextureCache, + cvPixelBufferRef, + nil, + &cvOpenGLTextureRef); + + // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image. + return CVOpenGLTextureGetName(cvOpenGLTextureRef); +} + +id<MTLTexture> AVFVideoFrameRenderer::createMetalTexture(id<MTLDevice> mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef, + MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef) +{ + CVReturn cvret; + // Create a Metal Core Video texture cache from the pixel buffer. + cvret = CVMetalTextureCacheCreate( + kCFAllocatorDefault, + nil, + mtlDevice, + nil, + &cvMetalTextureCacheRef); + + // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. + cvret = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + cvMetalTextureCacheRef, + cvPixelBufferRef, nil, + pixelFormat, + width, height, + 0, + &cvMetalTextureRef); + + // Get a Metal texture using the CoreVideo Metal texture reference. + return CVMetalTextureGetTexture(cvMetalTextureRef); +} diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index b60b276e9..174220f37 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -53,7 +53,8 @@ ios|tvos { } LIBS += -framework Foundation } else { - LIBS += -framework AppKit + INCLUDEPATH += $$[QT_INSTALL_HEADERS] + LIBS += -framework AppKit -framework Metal qtConfig(opengl) { HEADERS += \ diff --git a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp index 402ddf334..1b597a241 100644 --- a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp +++ b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp @@ -1244,14 +1244,14 @@ void tst_QVideoFrame::image_data() QTest::newRow("64x64 YUV420P") << QSize(64, 64) << QVideoFrame::Format_YUV420P - << 13288 + << 18432 << 256 << QImage::Format_ARGB32; QTest::newRow("64x64 YV12") << QSize(64, 64) << QVideoFrame::Format_YV12 - << 16384 + << 18432 << 256 << QImage::Format_ARGB32; @@ -1272,14 +1272,14 @@ void tst_QVideoFrame::image_data() QTest::newRow("64x64 NV12") << QSize(64, 64) << QVideoFrame::Format_NV12 - << 16384 + << 24576 << 256 << QImage::Format_ARGB32; QTest::newRow("64x64 NV21") << QSize(64, 64) << QVideoFrame::Format_NV21 - << 16384 + << 24576 << 256 << QImage::Format_ARGB32; #endif |