From 1994b277bfb5420afbbe814e9cb2ab47e1c45b78 Mon Sep 17 00:00:00 2001 From: Jani Heikkinen Date: Wed, 3 Mar 2021 15:05:56 +0200 Subject: Bump version Change-Id: I2b16e3e106a6bd672ffc092a68f8d39bc4b6c0a8 --- .qmake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.qmake.conf b/.qmake.conf index 5fb75cffe..163ffef25 100644 --- a/.qmake.conf +++ b/.qmake.conf @@ -2,4 +2,4 @@ load(qt_build_config) DEFINES += QT_NO_FOREACH QT_NO_JAVA_STYLE_ITERATORS QT_NO_LINKED_LIST -MODULE_VERSION = 5.15.3 +MODULE_VERSION = 5.15.4 -- cgit v1.2.3 From 1c9da622dc7e8ac6fa42f5263a5410098073b662 Mon Sep 17 00:00:00 2001 From: Pablo Luis Garcia Date: Mon, 19 Aug 2019 13:58:42 +0200 Subject: macOS: Fix video rendering on with OpenGL Core profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CoreAnimation OpenGL renderer only has support for pre 3.0 profiles. Changed CoreAnimation renderer to use a Metal texture for rendering and use a pixel buffer to share the texture data with an OpenGL texture, making it possible to render with 3.2+ profiles. Fixes: QTBUG-51064 Fixes: QTBUG-62694 Change-Id: I48a4a6e0d8fbc48170dfe82d1e71cd265d70179a Reviewed-by: Morten Johan Sørvig (cherry picked from commit abab792c4f0977a4599e5b7b2e97a5ee586fe388) Reviewed-by: Richard Moe Gustavsen --- .../mediaplayer/avfvideoframerenderer.h | 39 ++++ .../mediaplayer/avfvideoframerenderer.mm | 225 +++++++++++++++++++-- .../avfoundation/mediaplayer/mediaplayer.pro | 3 +- 3 files changed, 248 insertions(+), 19 deletions(-) diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index c450661da..28b47ac57 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -45,12 +45,16 @@ #include #include +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + @class CARenderer; @class AVPlayerLayer; QT_BEGIN_NAMESPACE class QOpenGLFramebufferObject; +class QOpenGLShaderProgram; class QWindow; class QOpenGLContext; class QAbstractVideoSurface; @@ -65,17 +69,52 @@ public: GLuint renderLayerToTexture(AVPlayerLayer *layer); QImage renderLayerToImage(AVPlayerLayer *layer); + static GLuint createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, + CVOpenGLTextureCacheRef cvglTextureCache, + CVPixelBufferRef cvPixelBufferRef, + CVOpenGLTextureRef cvOpenGLTextureRef); + + static id createMetalTexture(id mtlDevice, + CVMetalTextureCacheRef cvMetalTextureCacheRef, + CVPixelBufferRef cvPixelBufferRef, + MTLPixelFormat pixelFormat, size_t width, size_t height, + CVMetalTextureRef cvMetalTextureRef); + private: QOpenGLFramebufferObject* initRenderer(AVPlayerLayer *layer); void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); + void renderLayerToFBOCoreOpenGL(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); CARenderer *m_videoLayerRenderer; QAbstractVideoSurface *m_surface; QOpenGLFramebufferObject *m_fbo[2]; + QOpenGLShaderProgram *m_shader = nullptr; QWindow *m_offscreenSurface; QOpenGLContext *m_glContext; QSize m_targetSize; + bool m_useCoreProfile = false; + + // Shared pixel buffer + CVPixelBufferRef m_CVPixelBuffer; + + // OpenGL Texture + CVOpenGLTextureCacheRef m_CVGLTextureCache; + CVOpenGLTextureRef m_CVGLTexture; + CGLPixelFormatObj m_CGLPixelFormat; + GLuint m_textureName = 0; + + // Metal Texture + CVMetalTextureRef m_CVMTLTexture; + CVMetalTextureCacheRef m_CVMTLTextureCache; + id m_metalDevice = nil; + id m_metalTexture = nil; + + NSOpenGLContext *m_NSGLContext = nullptr; + + GLuint m_quadVao = 0; + GLuint m_quadVbos[2]; + uint m_currentBuffer; bool m_isContextShared; }; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 9a61e35df..4c7364a11 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -42,6 +42,8 @@ #include #include #include +#include +#include #ifdef QT_DEBUG_AVF #include @@ -76,6 +78,12 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() delete m_fbo[1]; delete m_offscreenSurface; delete m_glContext; + + if (m_useCoreProfile) { + glDeleteVertexArrays(1, &m_quadVao); + glDeleteBuffers(2, m_quadVbos); + delete m_shader; + } } GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) @@ -166,16 +174,124 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay [m_videoLayerRenderer release]; m_videoLayerRenderer = nullptr; } + + if (m_useCoreProfile) { + glDeleteVertexArrays(1, &m_quadVao); + glDeleteBuffers(2, m_quadVbos); + delete m_shader; + m_shader = nullptr; + } } //Need current context if (m_glContext) m_glContext->makeCurrent(m_offscreenSurface); - //Create the CARenderer if needed + if (!m_metalDevice) + m_metalDevice = MTLCreateSystemDefaultDevice(); + + if (@available(macOS 10.13, *)) { + m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() == + QSurfaceFormat::CoreProfile); + } else { + m_useCoreProfile = false; + } + + // Create the CARenderer if needed for no Core OpenGL if (!m_videoLayerRenderer) { - m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() options: nil]; - [m_videoLayerRenderer retain]; + if (!m_useCoreProfile) { + m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() + options: nil]; + [m_videoLayerRenderer retain]; + } else if (@available(macOS 10.13, *)) { + // This is always true when m_useCoreProfile is true, but the compiler wants the check + // anyway + // Setup Core OpenGL shader, VAO, VBOs and metal renderer + m_shader = new QOpenGLShaderProgram(); + m_shader->create(); + if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core + in vec2 qt_VertexPosition; + in vec2 qt_VertexTexCoord; + out vec2 qt_TexCoord; + void main() + { + qt_TexCoord = qt_VertexTexCoord; + gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f); + })")) { + qCritical() << "Vertex shader compilation failed" << m_shader->log(); + } + if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core + in vec2 qt_TexCoord; + out vec4 fragColor; + uniform sampler2DRect videoFrame; + void main(void) + { + ivec2 textureDim = textureSize(videoFrame); + fragColor = texture(videoFrame, qt_TexCoord * textureDim); + })")) { + qCritical() << "Fragment shader compilation failed" << m_shader->log(); + } + + // Setup quad where the video frame will be attached + GLfloat vertices[] = { + -1.0f, -1.0f, + 1.0f, -1.0f, + -1.0f, 1.0f, + 1.0f, 1.0f, + }; + + GLfloat uvs[] = { + 0.0f, 0.0f, + 1.0f, 0.0f, + 0.0f, 1.0f, + 1.0f, 1.0f, + }; + + glGenVertexArrays(1, &m_quadVao); + glBindVertexArray(m_quadVao); + + // Create vertex buffer objects for vertices + glGenBuffers(2, m_quadVbos); + + // Setup vertices + glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]); + glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); + glEnableVertexAttribArray(0); + + // Setup uvs + glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]); + glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW); + glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); + glEnableVertexAttribArray(1); + + glBindVertexArray(0); + + // Setup shared Metal/OpenGL pixel buffer and textures + m_NSGLContext = static_cast((QOpenGLContext::currentContext()->nativeHandle().data()))->context(); + m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj; + + NSDictionary* cvBufferProperties = @{ + static_cast(kCVPixelBufferOpenGLCompatibilityKey) : @YES, + static_cast(kCVPixelBufferMetalCompatibilityKey): @YES, + }; + + CVPixelBufferCreate(kCFAllocatorDefault, static_cast(m_targetSize.width()), + static_cast(m_targetSize.height()), kCVPixelFormatType_32BGRA, + static_cast(cvBufferProperties), &m_CVPixelBuffer); + + m_textureName = createGLTexture(reinterpret_cast(m_NSGLContext.CGLContextObj), + m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer, + m_CVGLTexture); + m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer, + MTLPixelFormatBGRA8Unorm, + static_cast(m_targetSize.width()), + static_cast(m_targetSize.height()), + m_CVMTLTexture); + + m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; + [m_videoLayerRenderer retain]; + } } //Set/Change render source if needed @@ -211,28 +327,101 @@ void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFrameb glViewport(0, 0, m_targetSize.width(), m_targetSize.height()); - glMatrixMode(GL_PROJECTION); - glPushMatrix(); - glLoadIdentity(); - - //Render to FBO with inverted Y - glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); - - glMatrixMode(GL_MODELVIEW); - glPushMatrix(); - glLoadIdentity(); + if (m_useCoreProfile) { + CGLLockContext(m_NSGLContext.CGLContextObj); + m_shader->bind(); + glBindVertexArray(m_quadVao); + } else { + glMatrixMode(GL_PROJECTION); + glPushMatrix(); + glLoadIdentity(); + + // Render to FBO with inverted Y + glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); + + glMatrixMode(GL_MODELVIEW); + glPushMatrix(); + glLoadIdentity(); + } - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr]; [m_videoLayerRenderer addUpdateRect:layer.bounds]; [m_videoLayerRenderer render]; [m_videoLayerRenderer endFrame]; - glMatrixMode(GL_MODELVIEW); - glPopMatrix(); - glMatrixMode(GL_PROJECTION); - glPopMatrix(); + if (m_useCoreProfile) { + glActiveTexture(0); + glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName); + + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + + glBindTexture(GL_TEXTURE_RECTANGLE, 0); + + glBindVertexArray(0); + + m_shader->release(); + + CGLFlushDrawable(m_NSGLContext.CGLContextObj); + CGLUnlockContext(m_NSGLContext.CGLContextObj); + } else { + glMatrixMode(GL_MODELVIEW); + glPopMatrix(); + glMatrixMode(GL_PROJECTION); + glPopMatrix(); + } glFinish(); //Rendering needs to be done before passing texture to video frame fbo->release(); } + +GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache, + CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef) +{ + CVReturn cvret; + // Create an OpenGL CoreVideo texture cache from the pixel buffer. + cvret = CVOpenGLTextureCacheCreate( + kCFAllocatorDefault, + nil, + cglContextObj, + cglPixelFormtObj, + nil, + &cvglTextureCache); + + // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache. + cvret = CVOpenGLTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + cvglTextureCache, + cvPixelBufferRef, + nil, + &cvOpenGLTextureRef); + + // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image. + return CVOpenGLTextureGetName(cvOpenGLTextureRef); +} + +id AVFVideoFrameRenderer::createMetalTexture(id mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef, + MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef) +{ + CVReturn cvret; + // Create a Metal Core Video texture cache from the pixel buffer. + cvret = CVMetalTextureCacheCreate( + kCFAllocatorDefault, + nil, + mtlDevice, + nil, + &cvMetalTextureCacheRef); + + // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. + cvret = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + cvMetalTextureCacheRef, + cvPixelBufferRef, nil, + pixelFormat, + width, height, + 0, + &cvMetalTextureRef); + + // Get a Metal texture using the CoreVideo Metal texture reference. + return CVMetalTextureGetTexture(cvMetalTextureRef); +} diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index b60b276e9..174220f37 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -53,7 +53,8 @@ ios|tvos { } LIBS += -framework Foundation } else { - LIBS += -framework AppKit + INCLUDEPATH += $$[QT_INSTALL_HEADERS] + LIBS += -framework AppKit -framework Metal qtConfig(opengl) { HEADERS += \ -- cgit v1.2.3 From 5298dcaa5d07410128594d7f37124f7d93d5dc86 Mon Sep 17 00:00:00 2001 From: Andy Shaw Date: Thu, 25 Mar 2021 12:14:30 +0100 Subject: AVFoundation: Only remove the video output if it is still attached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the video output is no longer attached then it will cause a crash if you try to remove it, so we need to check it is attached before removing. Change-Id: I90a119ae8e605ee88740248c94c7cea03acf4d50 Reviewed-by: Tor Arne Vestbø (cherry picked from commit 8b72da5f4b2d81444c731bc4e8eafca59e693bf6) Reviewed-by: Qt Cherry-pick Bot --- src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm index cad424018..75b73ae99 100644 --- a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm +++ b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm @@ -278,7 +278,8 @@ AVFCameraRendererControl::AVFCameraRendererControl(QObject *parent) AVFCameraRendererControl::~AVFCameraRendererControl() { - [m_cameraSession->captureSession() removeOutput:m_videoDataOutput]; + if ([m_cameraSession->captureSession().outputs containsObject:m_videoDataOutput]) + [m_cameraSession->captureSession() removeOutput:m_videoDataOutput]; [m_viewfinderFramesDelegate release]; if (m_delegateQueue) dispatch_release(m_delegateQueue); -- cgit v1.2.3 From 51282e083d7a1c06309cf7f2fc216f6203b4fbb2 Mon Sep 17 00:00:00 2001 From: Dmitry Shachnev Date: Sat, 20 Feb 2021 15:18:07 +0300 Subject: tst_qvideoframe: Increase buffer sizes used in image() tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YUV420P and YV12 formats use three planes which need 18432 bytes in total: - height × bytesPerLine = 64 × 256 = 16384 bytes; - height/2 × width/2 × uvPixelStride = 32 × 32 × 1 = 1024 bytes; - height/2 × width/2 × uvPixelStride = 32 × 32 × 1 = 1024 bytes. NV12 and NV21 formats use two planes which need 24576 bytes in total: - height × bytesPerLine = 64 × 256 = 16384 bytes; - height/2 × bytesPerLine = 32 × 256 = 8192 bytes. Fixes https://bugs.debian.org/982973. Change-Id: I68433dbb5370a6e4cdd2a85e2a9554fab642f5b6 Reviewed-by: Lars Knoll (cherry picked from commit 1c2a8655ac3dd5f8de5294a77ac5e585add82229) Reviewed-by: Qt Cherry-pick Bot --- tests/auto/unit/qvideoframe/tst_qvideoframe.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp index 402ddf334..1b597a241 100644 --- a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp +++ b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp @@ -1244,14 +1244,14 @@ void tst_QVideoFrame::image_data() QTest::newRow("64x64 YUV420P") << QSize(64, 64) << QVideoFrame::Format_YUV420P - << 13288 + << 18432 << 256 << QImage::Format_ARGB32; QTest::newRow("64x64 YV12") << QSize(64, 64) << QVideoFrame::Format_YV12 - << 16384 + << 18432 << 256 << QImage::Format_ARGB32; @@ -1272,14 +1272,14 @@ void tst_QVideoFrame::image_data() QTest::newRow("64x64 NV12") << QSize(64, 64) << QVideoFrame::Format_NV12 - << 16384 + << 24576 << 256 << QImage::Format_ARGB32; QTest::newRow("64x64 NV21") << QSize(64, 64) << QVideoFrame::Format_NV21 - << 16384 + << 24576 << 256 << QImage::Format_ARGB32; #endif -- cgit v1.2.3