summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.qmake.conf2
-rw-r--r--src/multimedia/playback/qmediaplayer.cpp2
-rw-r--r--src/multimedia/video/qvideoframe.cpp5
-rw-r--r--src/multimediawidgets/qpaintervideosurface.cpp3
-rw-r--r--src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm3
-rw-r--r--src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h39
-rw-r--r--src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm225
-rw-r--r--src/plugins/avfoundation/mediaplayer/mediaplayer.pro3
-rw-r--r--src/plugins/videonode/videonode.pro2
-rw-r--r--tests/auto/unit/qpaintervideosurface/tst_qpaintervideosurface.cpp4
-rw-r--r--tests/auto/unit/qvideoframe/tst_qvideoframe.cpp10
11 files changed, 262 insertions, 36 deletions
diff --git a/.qmake.conf b/.qmake.conf
index 5fb75cffe..163ffef25 100644
--- a/.qmake.conf
+++ b/.qmake.conf
@@ -2,4 +2,4 @@ load(qt_build_config)
DEFINES += QT_NO_FOREACH QT_NO_JAVA_STYLE_ITERATORS QT_NO_LINKED_LIST
-MODULE_VERSION = 5.15.3
+MODULE_VERSION = 5.15.4
diff --git a/src/multimedia/playback/qmediaplayer.cpp b/src/multimedia/playback/qmediaplayer.cpp
index 1b8c0df2d..382d8b30b 100644
--- a/src/multimedia/playback/qmediaplayer.cpp
+++ b/src/multimedia/playback/qmediaplayer.cpp
@@ -1349,7 +1349,7 @@ QStringList QMediaPlayer::supportedCustomAudioRoles() const
Defines the status of a media player's current media.
\value UnknownMediaStatus The status of the media cannot be determined.
- \value NoMedia The is no current media. The player is in the StoppedState.
+ \value NoMedia There is no current media. The player is in the StoppedState.
\value LoadingMedia The current media is being loaded. The player may be in any state.
\value LoadedMedia The current media has been loaded. The player is in the StoppedState.
\value StalledMedia Playback of the current media has stalled due to insufficient buffering or
diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 7840eda31..fd7b74075 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -964,8 +964,6 @@ QVideoFrame::PixelFormat QVideoFrame::pixelFormatFromImageFormat(QImage::Format
return Format_RGB555;
case QImage::Format_RGB888:
return Format_RGB24;
- case QImage::Format_Grayscale8:
- return Format_Y8;
default:
return Format_Invalid;
}
@@ -1019,13 +1017,12 @@ QImage::Format QVideoFrame::imageFormatFromPixelFormat(PixelFormat format)
case Format_IMC2:
case Format_IMC3:
case Format_IMC4:
+ case Format_Y8:
case Format_Y16:
case Format_Jpeg:
case Format_CameraRaw:
case Format_AdobeDng:
return QImage::Format_Invalid;
- case Format_Y8:
- return QImage::Format_Grayscale8;
case Format_User:
default:
return QImage::Format_Invalid;
diff --git a/src/multimediawidgets/qpaintervideosurface.cpp b/src/multimediawidgets/qpaintervideosurface.cpp
index 80bfec7bc..5fe76d869 100644
--- a/src/multimediawidgets/qpaintervideosurface.cpp
+++ b/src/multimediawidgets/qpaintervideosurface.cpp
@@ -109,8 +109,7 @@ QVideoSurfaceGenericPainter::QVideoSurfaceGenericPainter()
m_imagePixelFormats << QVideoFrame::Format_RGB24;
m_imagePixelFormats << QVideoFrame::Format_ARGB32
- << QVideoFrame::Format_RGB565
- << QVideoFrame::Format_Y8;
+ << QVideoFrame::Format_RGB565;
}
QList<QVideoFrame::PixelFormat> QVideoSurfaceGenericPainter::supportedPixelFormats(
diff --git a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm
index 9e1bf3f84..7bf9de071 100644
--- a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm
+++ b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm
@@ -278,7 +278,8 @@ AVFCameraRendererControl::AVFCameraRendererControl(QObject *parent)
AVFCameraRendererControl::~AVFCameraRendererControl()
{
- [m_cameraSession->captureSession() removeOutput:m_videoDataOutput];
+ if ([m_cameraSession->captureSession().outputs containsObject:m_videoDataOutput])
+ [m_cameraSession->captureSession() removeOutput:m_videoDataOutput];
[m_viewfinderFramesDelegate release];
if (m_delegateQueue)
dispatch_release(m_delegateQueue);
diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
index 99b6bb0b5..c5d7fe66c 100644
--- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
+++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
@@ -45,12 +45,16 @@
#include <QtGui/QOpenGLContext>
#include <QtCore/QSize>
+#import "Metal/Metal.h"
+#import "MetalKit/MetalKit.h"
+
@class CARenderer;
@class AVPlayerLayer;
QT_BEGIN_NAMESPACE
class QOpenGLFramebufferObject;
+class QOpenGLShaderProgram;
class QWindow;
class QOpenGLContext;
class QAbstractVideoSurface;
@@ -65,17 +69,52 @@ public:
GLuint renderLayerToTexture(AVPlayerLayer *layer);
QImage renderLayerToImage(AVPlayerLayer *layer);
+ static GLuint createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj,
+ CVOpenGLTextureCacheRef cvglTextureCache,
+ CVPixelBufferRef cvPixelBufferRef,
+ CVOpenGLTextureRef cvOpenGLTextureRef);
+
+ static id<MTLTexture> createMetalTexture(id<MTLDevice> mtlDevice,
+ CVMetalTextureCacheRef cvMetalTextureCacheRef,
+ CVPixelBufferRef cvPixelBufferRef,
+ MTLPixelFormat pixelFormat, size_t width, size_t height,
+ CVMetalTextureRef cvMetalTextureRef);
+
private:
QOpenGLFramebufferObject* initRenderer(AVPlayerLayer *layer);
void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo);
+ void renderLayerToFBOCoreOpenGL(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo);
CARenderer *m_videoLayerRenderer;
QAbstractVideoSurface *m_surface;
QOpenGLFramebufferObject *m_fbo[2];
+ QOpenGLShaderProgram *m_shader = nullptr;
QWindow *m_offscreenSurface;
QOpenGLContext *m_glContext;
QSize m_targetSize;
+ bool m_useCoreProfile = false;
+
+ // Shared pixel buffer
+ CVPixelBufferRef m_CVPixelBuffer;
+
+ // OpenGL Texture
+ CVOpenGLTextureCacheRef m_CVGLTextureCache;
+ CVOpenGLTextureRef m_CVGLTexture;
+ CGLPixelFormatObj m_CGLPixelFormat;
+ GLuint m_textureName = 0;
+
+ // Metal Texture
+ CVMetalTextureRef m_CVMTLTexture;
+ CVMetalTextureCacheRef m_CVMTLTextureCache;
+ id<MTLDevice> m_metalDevice = nil;
+ id<MTLTexture> m_metalTexture = nil;
+
+ NSOpenGLContext *m_NSGLContext = nullptr;
+
+ GLuint m_quadVao = 0;
+ GLuint m_quadVbos[2];
+
uint m_currentBuffer;
bool m_isContextShared;
};
diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
index 51f961729..23e5497a2 100644
--- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
+++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
@@ -42,6 +42,8 @@
#include <QtMultimedia/qabstractvideosurface.h>
#include <QtGui/QOpenGLFramebufferObject>
#include <QtGui/QWindow>
+#include <QOpenGLShaderProgram>
+#include <QtPlatformHeaders/QCocoaNativeContext>
#ifdef QT_DEBUG_AVF
#include <QtCore/qdebug.h>
@@ -76,6 +78,12 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer()
delete m_fbo[1];
delete m_offscreenSurface;
delete m_glContext;
+
+ if (m_useCoreProfile) {
+ glDeleteVertexArrays(1, &m_quadVao);
+ glDeleteBuffers(2, m_quadVbos);
+ delete m_shader;
+ }
}
GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer)
@@ -166,16 +174,124 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay
[m_videoLayerRenderer release];
m_videoLayerRenderer = nullptr;
}
+
+ if (m_useCoreProfile) {
+ glDeleteVertexArrays(1, &m_quadVao);
+ glDeleteBuffers(2, m_quadVbos);
+ delete m_shader;
+ m_shader = nullptr;
+ }
}
//Need current context
if (m_glContext)
m_glContext->makeCurrent(m_offscreenSurface);
- //Create the CARenderer if needed
+ if (!m_metalDevice)
+ m_metalDevice = MTLCreateSystemDefaultDevice();
+
+ if (@available(macOS 10.13, *)) {
+ m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() ==
+ QSurfaceFormat::CoreProfile);
+ } else {
+ m_useCoreProfile = false;
+ }
+
+ // Create the CARenderer if needed for no Core OpenGL
if (!m_videoLayerRenderer) {
- m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() options: nil];
- [m_videoLayerRenderer retain];
+ if (!m_useCoreProfile) {
+ m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext()
+ options: nil];
+ [m_videoLayerRenderer retain];
+ } else if (@available(macOS 10.13, *)) {
+ // This is always true when m_useCoreProfile is true, but the compiler wants the check
+ // anyway
+ // Setup Core OpenGL shader, VAO, VBOs and metal renderer
+ m_shader = new QOpenGLShaderProgram();
+ m_shader->create();
+ if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core
+ in vec2 qt_VertexPosition;
+ in vec2 qt_VertexTexCoord;
+ out vec2 qt_TexCoord;
+ void main()
+ {
+ qt_TexCoord = qt_VertexTexCoord;
+ gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f);
+ })")) {
+ qCritical() << "Vertex shader compilation failed" << m_shader->log();
+ }
+ if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core
+ in vec2 qt_TexCoord;
+ out vec4 fragColor;
+ uniform sampler2DRect videoFrame;
+ void main(void)
+ {
+ ivec2 textureDim = textureSize(videoFrame);
+ fragColor = texture(videoFrame, qt_TexCoord * textureDim);
+ })")) {
+ qCritical() << "Fragment shader compilation failed" << m_shader->log();
+ }
+
+ // Setup quad where the video frame will be attached
+ GLfloat vertices[] = {
+ -1.0f, -1.0f,
+ 1.0f, -1.0f,
+ -1.0f, 1.0f,
+ 1.0f, 1.0f,
+ };
+
+ GLfloat uvs[] = {
+ 0.0f, 0.0f,
+ 1.0f, 0.0f,
+ 0.0f, 1.0f,
+ 1.0f, 1.0f,
+ };
+
+ glGenVertexArrays(1, &m_quadVao);
+ glBindVertexArray(m_quadVao);
+
+ // Create vertex buffer objects for vertices
+ glGenBuffers(2, m_quadVbos);
+
+ // Setup vertices
+ glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]);
+ glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+ glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr);
+ glEnableVertexAttribArray(0);
+
+ // Setup uvs
+ glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]);
+ glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW);
+ glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr);
+ glEnableVertexAttribArray(1);
+
+ glBindVertexArray(0);
+
+ // Setup shared Metal/OpenGL pixel buffer and textures
+ m_NSGLContext = static_cast<QCocoaNativeContext*>((QOpenGLContext::currentContext()->nativeHandle().data()))->context();
+ m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj;
+
+ NSDictionary* cvBufferProperties = @{
+ static_cast<NSString*>(kCVPixelBufferOpenGLCompatibilityKey) : @YES,
+ static_cast<NSString*>(kCVPixelBufferMetalCompatibilityKey): @YES,
+ };
+
+ CVPixelBufferCreate(kCFAllocatorDefault, static_cast<size_t>(m_targetSize.width()),
+ static_cast<size_t>(m_targetSize.height()), kCVPixelFormatType_32BGRA,
+ static_cast<CFDictionaryRef>(cvBufferProperties), &m_CVPixelBuffer);
+
+ m_textureName = createGLTexture(reinterpret_cast<CGLContextObj>(m_NSGLContext.CGLContextObj),
+ m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer,
+ m_CVGLTexture);
+ m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer,
+ MTLPixelFormatBGRA8Unorm,
+ static_cast<size_t>(m_targetSize.width()),
+ static_cast<size_t>(m_targetSize.height()),
+ m_CVMTLTexture);
+
+ m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil];
+ [m_videoLayerRenderer retain];
+ }
}
//Set/Change render source if needed
@@ -211,28 +327,101 @@ void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFrameb
glViewport(0, 0, m_targetSize.width(), m_targetSize.height());
- glMatrixMode(GL_PROJECTION);
- glPushMatrix();
- glLoadIdentity();
-
- //Render to FBO with inverted Y
- glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0);
-
- glMatrixMode(GL_MODELVIEW);
- glPushMatrix();
- glLoadIdentity();
+ if (m_useCoreProfile) {
+ CGLLockContext(m_NSGLContext.CGLContextObj);
+ m_shader->bind();
+ glBindVertexArray(m_quadVao);
+ } else {
+ glMatrixMode(GL_PROJECTION);
+ glPushMatrix();
+ glLoadIdentity();
+
+ // Render to FBO with inverted Y
+ glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0);
+
+ glMatrixMode(GL_MODELVIEW);
+ glPushMatrix();
+ glLoadIdentity();
+ }
- [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL];
+ [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr];
[m_videoLayerRenderer addUpdateRect:layer.bounds];
[m_videoLayerRenderer render];
[m_videoLayerRenderer endFrame];
- glMatrixMode(GL_MODELVIEW);
- glPopMatrix();
- glMatrixMode(GL_PROJECTION);
- glPopMatrix();
+ if (m_useCoreProfile) {
+ glActiveTexture(0);
+ glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName);
+
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+ glBindTexture(GL_TEXTURE_RECTANGLE, 0);
+
+ glBindVertexArray(0);
+
+ m_shader->release();
+
+ CGLFlushDrawable(m_NSGLContext.CGLContextObj);
+ CGLUnlockContext(m_NSGLContext.CGLContextObj);
+ } else {
+ glMatrixMode(GL_MODELVIEW);
+ glPopMatrix();
+ glMatrixMode(GL_PROJECTION);
+ glPopMatrix();
+ }
glFinish(); //Rendering needs to be done before passing texture to video frame
fbo->release();
}
+
+GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache,
+ CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef)
+{
+ CVReturn cvret;
+ // Create an OpenGL CoreVideo texture cache from the pixel buffer.
+ cvret = CVOpenGLTextureCacheCreate(
+ kCFAllocatorDefault,
+ nil,
+ cglContextObj,
+ cglPixelFormtObj,
+ nil,
+ &cvglTextureCache);
+
+ // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache.
+ cvret = CVOpenGLTextureCacheCreateTextureFromImage(
+ kCFAllocatorDefault,
+ cvglTextureCache,
+ cvPixelBufferRef,
+ nil,
+ &cvOpenGLTextureRef);
+
+ // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image.
+ return CVOpenGLTextureGetName(cvOpenGLTextureRef);
+}
+
+id<MTLTexture> AVFVideoFrameRenderer::createMetalTexture(id<MTLDevice> mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef,
+ MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef)
+{
+ CVReturn cvret;
+ // Create a Metal Core Video texture cache from the pixel buffer.
+ cvret = CVMetalTextureCacheCreate(
+ kCFAllocatorDefault,
+ nil,
+ mtlDevice,
+ nil,
+ &cvMetalTextureCacheRef);
+
+ // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache.
+ cvret = CVMetalTextureCacheCreateTextureFromImage(
+ kCFAllocatorDefault,
+ cvMetalTextureCacheRef,
+ cvPixelBufferRef, nil,
+ pixelFormat,
+ width, height,
+ 0,
+ &cvMetalTextureRef);
+
+ // Get a Metal texture using the CoreVideo Metal texture reference.
+ return CVMetalTextureGetTexture(cvMetalTextureRef);
+}
diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro
index b60b276e9..174220f37 100644
--- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro
+++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro
@@ -53,7 +53,8 @@ ios|tvos {
}
LIBS += -framework Foundation
} else {
- LIBS += -framework AppKit
+ INCLUDEPATH += $$[QT_INSTALL_HEADERS]
+ LIBS += -framework AppKit -framework Metal
qtConfig(opengl) {
HEADERS += \
diff --git a/src/plugins/videonode/videonode.pro b/src/plugins/videonode/videonode.pro
index ab7295406..889862c36 100644
--- a/src/plugins/videonode/videonode.pro
+++ b/src/plugins/videonode/videonode.pro
@@ -1,7 +1,7 @@
TEMPLATE = subdirs
QT_FOR_CONFIG += gui-private multimedia-private
-qtConfig(gpu_vivante) {
+qtConfig(gpu_vivante):qtConfig(gstreamer) {
SUBDIRS += imx6
}
diff --git a/tests/auto/unit/qpaintervideosurface/tst_qpaintervideosurface.cpp b/tests/auto/unit/qpaintervideosurface/tst_qpaintervideosurface.cpp
index 98ec5fc06..be5c85ab9 100644
--- a/tests/auto/unit/qpaintervideosurface/tst_qpaintervideosurface.cpp
+++ b/tests/auto/unit/qpaintervideosurface/tst_qpaintervideosurface.cpp
@@ -253,8 +253,8 @@ void tst_QPainterVideoSurface::supportedFormat_data()
<< QAbstractVideoBuffer::NoHandle
<< QVideoFrame::Format_Y8
<< QSize(640, 480)
- << true
- << true;
+ << false
+ << false;
QTest::newRow("Texture: rgb32 640x480")
<< QAbstractVideoBuffer::GLTextureHandle
<< QVideoFrame::Format_RGB32
diff --git a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp
index e99c4bf77..1b597a241 100644
--- a/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp
+++ b/tests/auto/unit/qvideoframe/tst_qvideoframe.cpp
@@ -978,7 +978,7 @@ void tst_QVideoFrame::formatConversion_data()
<< QImage::Format_Invalid
<< QVideoFrame::Format_IMC4;
QTest::newRow("QVideoFrame::Format_Y8")
- << QImage::Format_Grayscale8
+ << QImage::Format_Invalid
<< QVideoFrame::Format_Y8;
QTest::newRow("QVideoFrame::Format_Y16")
<< QImage::Format_Invalid
@@ -1244,14 +1244,14 @@ void tst_QVideoFrame::image_data()
QTest::newRow("64x64 YUV420P")
<< QSize(64, 64)
<< QVideoFrame::Format_YUV420P
- << 13288
+ << 18432
<< 256
<< QImage::Format_ARGB32;
QTest::newRow("64x64 YV12")
<< QSize(64, 64)
<< QVideoFrame::Format_YV12
- << 16384
+ << 18432
<< 256
<< QImage::Format_ARGB32;
@@ -1272,14 +1272,14 @@ void tst_QVideoFrame::image_data()
QTest::newRow("64x64 NV12")
<< QSize(64, 64)
<< QVideoFrame::Format_NV12
- << 16384
+ << 24576
<< 256
<< QImage::Format_ARGB32;
QTest::newRow("64x64 NV21")
<< QSize(64, 64)
<< QVideoFrame::Format_NV21
- << 16384
+ << 24576
<< 256
<< QImage::Format_ARGB32;
#endif