summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.qmake.conf2
-rw-r--r--src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm3
-rw-r--r--src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h39
-rw-r--r--src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm225
-rw-r--r--src/plugins/avfoundation/mediaplayer/mediaplayer.pro3
-rw-r--r--tests/auto/unit/qvideoframe/tst_qvideoframe.cpp8
6 files changed, 255 insertions, 25 deletions
diff --git a/.qmake.conf b/.qmake.conf
index 5fb75cffe..e62ae21d9 100644
--- a/.qmake.conf
+++ b/.qmake.conf
@@ -2,4 +2,4 @@ load(qt_build_config)
DEFINES += QT_NO_FOREACH QT_NO_JAVA_STYLE_ITERATORS QT_NO_LINKED_LIST
-MODULE_VERSION = 5.15.3
+MODULE_VERSION = 5.15.5
diff --git a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm
index 9e1bf3f84..7bf9de071 100644
--- a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm
+++ b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm
@@ -278,7 +278,8 @@ AVFCameraRendererControl::AVFCameraRendererControl(QObject *parent)
AVFCameraRendererControl::~AVFCameraRendererControl()
{
- [m_cameraSession->captureSession() removeOutput:m_videoDataOutput];
+ if ([m_cameraSession->captureSession().outputs containsObject:m_videoDataOutput])
+ [m_cameraSession->captureSession() removeOutput:m_videoDataOutput];
[m_viewfinderFramesDelegate release];
if (m_delegateQueue)
dispatch_release(m_delegateQueue);
diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
index 99b6bb0b5..c5d7fe66c 100644
--- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
+++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
@@ -45,12 +45,16 @@
#include <QtGui/QOpenGLContext>
#include <QtCore/QSize>
+#import "Metal/Metal.h"
+#import "MetalKit/MetalKit.h"
+
@class CARenderer;
@class AVPlayerLayer;
QT_BEGIN_NAMESPACE
class QOpenGLFramebufferObject;
+class QOpenGLShaderProgram;
class QWindow;
class QOpenGLContext;
class QAbstractVideoSurface;
@@ -65,17 +69,52 @@ public:
GLuint renderLayerToTexture(AVPlayerLayer *layer);
QImage renderLayerToImage(AVPlayerLayer *layer);
+ static GLuint createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj,
+ CVOpenGLTextureCacheRef cvglTextureCache,
+ CVPixelBufferRef cvPixelBufferRef,
+ CVOpenGLTextureRef cvOpenGLTextureRef);
+
+ static id<MTLTexture> createMetalTexture(id<MTLDevice> mtlDevice,
+ CVMetalTextureCacheRef cvMetalTextureCacheRef,
+ CVPixelBufferRef cvPixelBufferRef,
+ MTLPixelFormat pixelFormat, size_t width, size_t height,
+ CVMetalTextureRef cvMetalTextureRef);
+
private:
QOpenGLFramebufferObject* initRenderer(AVPlayerLayer *layer);
void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo);
+ void renderLayerToFBOCoreOpenGL(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo);
CARenderer *m_videoLayerRenderer;
QAbstractVideoSurface *m_surface;
QOpenGLFramebufferObject *m_fbo[2];
+ QOpenGLShaderProgram *m_shader = nullptr;
QWindow *m_offscreenSurface;
QOpenGLContext *m_glContext;
QSize m_targetSize;
+ bool m_useCoreProfile = false;
+
+ // Shared pixel buffer
+ CVPixelBufferRef m_CVPixelBuffer;
+
+ // OpenGL Texture
+ CVOpenGLTextureCacheRef m_CVGLTextureCache;
+ CVOpenGLTextureRef m_CVGLTexture;
+ CGLPixelFormatObj m_CGLPixelFormat;
+ GLuint m_textureName = 0;
+
+ // Metal Texture
+ CVMetalTextureRef m_CVMTLTexture;
+ CVMetalTextureCacheRef m_CVMTLTextureCache;
+ id<MTLDevice> m_metalDevice = nil;
+ id<MTLTexture> m_metalTexture = nil;
+
+ NSOpenGLContext *m_NSGLContext = nullptr;
+
+ GLuint m_quadVao = 0;
+ GLuint m_quadVbos[2];
+
uint m_currentBuffer;
bool m_isContextShared;
};
diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
index 51f961729..23e5497a2 100644
--- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
+++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
@@ -42,6 +42,8 @@
#include <QtMultimedia/qabstractvideosurface.h>
#include <QtGui/QOpenGLFramebufferObject>
#include <QtGui/QWindow>
+#include <QOpenGLShaderProgram>
+#include <QtPlatformHeaders/QCocoaNativeContext>
#ifdef QT_DEBUG_AVF
#include <QtCore/qdebug.h>
@@ -76,6 +78,12 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer()
delete m_fbo[1];
delete m_offscreenSurface;
delete m_glContext;
+
+ if (m_useCoreProfile) {
+ glDeleteVertexArrays(1, &m_quadVao);
+ glDeleteBuffers(2, m_quadVbos);
+ delete m_shader;
+ }
}
GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer)
@@ -166,16 +174,124 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay
[m_videoLayerRenderer release];
m_videoLayerRenderer = nullptr;
}
+
+ if (m_useCoreProfile) {
+ glDeleteVertexArrays(1, &m_quadVao);
+ glDeleteBuffers(2, m_quadVbos);
+ delete m_shader;
+ m_shader = nullptr;
+ }
}
//Need current context
if (m_glContext)
m_glContext->makeCurrent(m_offscreenSurface);
- //Create the CARenderer if needed
+ if (!m_metalDevice)
+ m_metalDevice = MTLCreateSystemDefaultDevice();
+
+ if (@available(macOS 10.13, *)) {
+ m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() ==
+ QSurfaceFormat::CoreProfile);
+ } else {
+ m_useCoreProfile = false;
+ }
+
+ // Create the CARenderer if needed for no Core OpenGL
if (!m_videoLayerRenderer) {
- m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() options: nil];
- [m_videoLayerRenderer retain];
+ if (!m_useCoreProfile) {
+ m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext()
+ options: nil];
+ [m_videoLayerRenderer retain];
+ } else if (@available(macOS 10.13, *)) {
+ // This is always true when m_useCoreProfile is true, but the compiler wants the check
+ // anyway
+ // Setup Core OpenGL shader, VAO, VBOs and metal renderer
+ m_shader = new QOpenGLShaderProgram();
+ m_shader->create();
+ if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core
+ in vec2 qt_VertexPosition;
+ in vec2 qt_VertexTexCoord;
+ out vec2 qt_TexCoord;
+ void main()
+ {
+ qt_TexCoord = qt_VertexTexCoord;
+ gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f);
+ })")) {
+ qCritical() << "Vertex shader compilation failed" << m_shader->log();
+ }
+ if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core
+ in vec2 qt_TexCoord;
+ out vec4 fragColor;
+ uniform sampler2DRect videoFrame;
+ void main(void)
+ {
+ ivec2 textureDim = textureSize(videoFrame);
+ fragColor = texture(videoFrame, qt_TexCoord * textureDim);
+ })")) {
+ qCritical() << "Fragment shader compilation failed" << m_shader->log();
+ }
+
+ // Setup quad where the video frame will be attached
+ GLfloat vertices[] = {
+ -1.0f, -1.0f,
+ 1.0f, -1.0f,
+ -1.0f, 1.0f,
+ 1.0f, 1.0f,
+ };
+
+ GLfloat uvs[] = {
+ 0.0f, 0.0f,
+ 1.0f, 0.0f,
+ 0.0f, 1.0f,
+ 1.0f, 1.0f,
+ };
+
+ glGenVertexArrays(1, &m_quadVao);
+ glBindVertexArray(m_quadVao);
+
+ // Create vertex buffer objects for vertices
+ glGenBuffers(2, m_quadVbos);
+
+ // Setup vertices
+ glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]);
+ glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+ glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr);
+ glEnableVertexAttribArray(0);
+
+ // Setup uvs
+ glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]);
+ glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW);
+ glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr);
+ glEnableVertexAttribArray(1);
+
+ glBindVertexArray(0);
+
+ // Setup shared Metal/OpenGL pixel buffer and textures
+ m_NSGLContext = static_cast<QCocoaNativeContext*>((QOpenGLContext::currentContext()->nativeHandle().data()))->context();
+ m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj;
+
+ NSDictionary* cvBufferProperties = @{
+ static_cast<NSString*>(kCVPixelBufferOpenGLCompatibilityKey) : @YES,
+ static_cast<NSString*>(kCVPixelBufferMetalCompatibilityKey): @YES,
+ };
+
+ CVPixelBufferCreate(kCFAllocatorDefault, static_cast<size_t>(m_targetSize.width()),
+ static_cast<size_t>(m_targetSize.height()), kCVPixelFormatType_32BGRA,
+ static_cast<CFDictionaryRef>(cvBufferProperties), &m_CVPixelBuffer);
+
+ m_textureName = createGLTexture(reinterpret_cast<CGLContextObj>(m_NSGLContext.CGLContextObj),
+ m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer,
+ m_CVGLTexture);
+ m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer,
+ MTLPixelFormatBGRA8Unorm,
+ static_cast<size_t>(m_targetSize.width()),
+ static_cast<size_t>(m_targetSize.height()),
+ m_CVMTLTexture);
+
+ m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil];
+ [m_videoLayerRenderer retain];
+ }
}
//Set/Change render source if needed
@@ -211,28 +327,101 @@ void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFrameb
glViewport(0, 0, m_targetSize.width(), m_targetSize.height());
- glMatrixMode(GL_PROJECTION);
- glPushMatrix();
- glLoadIdentity();
-
- //Render to FBO with inverted Y
- glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0);
-
- glMatrixMode(GL_MODELVIEW);
- glPushMatrix();
- glLoadIdentity();
+ if (m_useCoreProfile) {
+ CGLLockContext(m_NSGLContext.CGLContextObj);
+ m_shader->bind();
+ glBindVertexArray(m_quadVao);
+ } else {
+ glMatrixMode(GL_PROJECTION);
+ glPushMatrix();
+ glLoadIdentity();
+
+ // Render to FBO with inverted Y
+ glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0);
+
+ glMatrixMode(GL_MODELVIEW);
+ glPushMatrix();
+ glLoadIdentity();
+ }
- [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL];
+ [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr];
[m_videoLayerRenderer addUpdateRect:layer.bounds];
[m_videoLayerRenderer render];
[m_videoLayerRenderer endFrame];
- glMatrixMode(GL_MODELVIEW);
- glPopMatrix();
- glMatrixMode(GL_PROJECTION);
- glPopMatrix();
+ if (m_useCoreProfile) {
+ glActiveTexture(0);
+ glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName);
+
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+ glBindTexture(GL_TEXTURE_RECTANGLE, 0);
+
+ glBindVertexArray(0);
+
+ m_shader->release();
+
+ CGLFlushDrawable(m_NSGLContext.CGLContextObj);
+ CGLUnlockContext(m_NSGLContext.CGLContextObj);
+ } else {
+ glMatrixMode(GL_MODELVIEW);
+ glPopMatrix();
+ glMatrixMode(GL_PROJECTION);
+ glPopMatrix();
+ }
glFinish(); //Rendering needs to be done before passing texture to video frame
fbo->release();
}
+
+GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache,
+ CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef)
+{
+ CVReturn cvret;
+ // Create an OpenGL CoreVideo texture cache from the pixel buffer.
+ cvret = CVOpenGLTextureCacheCreate(
+ kCFAllocatorDefault,
+ nil,
+ cglContextObj,
+ cglPixelFormtObj,
+ nil,
+ &cvglTextureCache);
+
+ // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache.
+ cvret = CVOpenGLTextureCacheCreateTextureFromImage(
+ kCFAllocatorDefault,
+ cvglTextureCache,
+ cvPixelBufferRef,
+ nil,
+ &cvOpenGLTextureRef);
+
+ // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image.
+ return CVOpenGLTextureGetName(cvOpenGLTextureRef);
+}
+
+id<MTLTexture> AVFVideoFrameRenderer::createMetalTexture(id<MTLDevice> mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef,
+ MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef)
+{
+ CVReturn cvret;
+ // Create a Metal Core Video texture cache from the pixel buffer.
+ cvret = CVMetalTextureCacheCreate(
+ kCFAllocatorDefault,
+ nil,
+ mtlDevice,
+ nil,
+ &cvMetalTextureCacheRef);
+
+ // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache.
+ cvret = CVMetalTextureCacheCreateTextureFromImage(
+ kCFAllocatorDefault,
+ cvMetalTextureCacheRef,
+ cvPixelBufferRef, nil,
+ pixelFormat,
+ width, height,
+ 0,
+ &cvMetalTextureRef);
+
+ // Get a Metal texture using the CoreVideo Metal texture reference.
+ return CVMetalTextureGetTexture(cvMetalTextureRef);
+}
diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro
index b60b276e9..174220f37 100644
--- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro
+++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro
@@ -53,7 +53,8 @@ ios|tvos {
}
LIBS += -framework Foundation
} else {
- LIBS += -framework AppKit
+ INCLUDEPATH += $$[QT_INSTALL_HEADERS]
+ LIBS += -framework AppKit -framework Metal
qtConfig(opengl) {
HEADERS += \
diff --git a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp
index 402ddf334..1b597a241 100644
--- a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp
+++ b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp
@@ -1244,14 +1244,14 @@ void tst_QVideoFrame::image_data()
QTest::newRow("64x64 YUV420P")
<< QSize(64, 64)
<< QVideoFrame::Format_YUV420P
- << 13288
+ << 18432
<< 256
<< QImage::Format_ARGB32;
QTest::newRow("64x64 YV12")
<< QSize(64, 64)
<< QVideoFrame::Format_YV12
- << 16384
+ << 18432
<< 256
<< QImage::Format_ARGB32;
@@ -1272,14 +1272,14 @@ void tst_QVideoFrame::image_data()
QTest::newRow("64x64 NV12")
<< QSize(64, 64)
<< QVideoFrame::Format_NV12
- << 16384
+ << 24576
<< 256
<< QImage::Format_ARGB32;
QTest::newRow("64x64 NV21")
<< QSize(64, 64)
<< QVideoFrame::Format_NV21
- << 16384
+ << 24576
<< 256
<< QImage::Format_ARGB32;
#endif