diff options
14 files changed, 415 insertions, 1285 deletions
diff --git a/src/imports/multimedia/qmldir b/src/imports/multimedia/qmldir index 48cdb0c8c..3444aaa06 100644 --- a/src/imports/multimedia/qmldir +++ b/src/imports/multimedia/qmldir @@ -5,4 +5,3 @@ typeinfo plugins.qmltypes prefer :/qt-project.org/imports/QtMultimedia/ typeinfo plugins.qmltypes Video 5.0 Video.qml - diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt index 29aca7773..063da1340 100644 --- a/src/multimedia/CMakeLists.txt +++ b/src/multimedia/CMakeLists.txt @@ -353,9 +353,12 @@ qt_internal_extend_target(Multimedia CONDITION APPLE AND NOT WATCHOS platform/darwin/audio/qcoreaudioinput.mm platform/darwin/audio/qcoreaudioinput_p.h platform/darwin/audio/qcoreaudiooutput.mm platform/darwin/audio/qcoreaudiooutput_p.h platform/darwin/audio/qcoreaudioutils.mm platform/darwin/audio/qcoreaudioutils_p.h + platform/darwin/mediaplayer/avfdisplaylink.mm platform/darwin/mediaplayer/avfdisplaylink_p.h platform/darwin/mediaplayer/avfmediaplayer.mm platform/darwin/mediaplayer/avfmediaplayer_p.h platform/darwin/mediaplayer/avfmetadata.mm platform/darwin/mediaplayer/avfmetadata_p.h + platform/darwin/mediaplayer/avfvideorenderercontrol.mm platform/darwin/mediaplayer/avfvideorenderercontrol_p.h platform/darwin/avfvideosink.mm platform/darwin/avfvideosink_p.h + platform/darwin/avfvideobuffer.mm platform/darwin/avfvideobuffer_p.h platform/darwin/qdarwindevicemanager.mm platform/darwin/qdarwindevicemanager_p.h platform/darwin/qdarwinformatsinfo.mm platform/darwin/qdarwinformatsinfo_p.h platform/darwin/qdarwinintegration.cpp platform/darwin/qdarwinintegration_p.h @@ -421,20 +424,6 @@ qt_internal_extend_target(Multimedia CONDITION APPLE AND NOT IOS AND NOT TVOS AN ${FWAudioUnit} ) -qt_internal_extend_target(Multimedia CONDITION APPLE AND QT_FEATURE_opengl AND NOT WATCHOS AND (IOS OR TVOS) - SOURCES - platform/darwin/mediaplayer/avfdisplaylink.mm platform/darwin/mediaplayer/avfdisplaylink_p.h - platform/darwin/mediaplayer/avfvideoframerenderer_ios.mm platform/darwin/mediaplayer/avfvideoframerenderer_ios_p.h - platform/darwin/mediaplayer/avfvideorenderercontrol.mm platform/darwin/mediaplayer/avfvideorenderercontrol_p.h -) - -qt_internal_extend_target(Multimedia CONDITION APPLE AND QT_FEATURE_opengl AND NOT IOS AND NOT TVOS AND NOT WATCHOS - SOURCES - platform/darwin/mediaplayer/avfdisplaylink.mm platform/darwin/mediaplayer/avfdisplaylink_p.h - platform/darwin/mediaplayer/avfvideoframerenderer.mm platform/darwin/mediaplayer/avfvideoframerenderer_p.h - platform/darwin/mediaplayer/avfvideorenderercontrol.mm platform/darwin/mediaplayer/avfvideorenderercontrol_p.h -) - qt_internal_extend_target(Multimedia CONDITION QNX SOURCES platform/qnx/audio/neutrinoserviceplugin.cpp platform/qnx/audio/neutrinoserviceplugin_p.h diff --git a/src/multimedia/platform/darwin/avfvideobuffer.mm b/src/multimedia/platform/darwin/avfvideobuffer.mm new file mode 100644 index 000000000..67bdaaa31 --- /dev/null +++ b/src/multimedia/platform/darwin/avfvideobuffer.mm @@ -0,0 +1,223 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "avfvideobuffer_p.h" +#include <private/qrhi_p.h> +#include <private/qrhimetal_p.h> +#include <private/qrhigles2_p.h> +#include <CoreVideo/CVMetalTexture.h> +#include <CoreVideo/CVMetalTextureCache.h> +#include <QtGui/qopenglcontext.h> + +#import <AVFoundation/AVFoundation.h> +#import <Metal/Metal.h> + +QT_USE_NAMESPACE + +AVFVideoBuffer::AVFVideoBuffer(QRhi *rhi, CVImageBufferRef buffer) + : QAbstractVideoBuffer(rhi ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle), + rhi(rhi), + m_buffer(buffer) +{ +// m_type = QVideoFrame::NoHandle; +// qDebug() << "RHI" << rhi; + CVPixelBufferRetain(m_buffer); +} + +AVFVideoBuffer::~AVFVideoBuffer() +{ + AVFVideoBuffer::unmap(); + if (cvMetalTexture) + CFRelease(cvMetalTexture); + if (cvMetalTextureCache) + CFRelease(cvMetalTextureCache); + if (cvOpenGLTexture) + CFRelease(cvOpenGLTexture); + if (cvOpenGLTextureCache) + CFRelease(cvOpenGLTextureCache); + CVPixelBufferRelease(m_buffer); +} + +AVFVideoBuffer::MapData AVFVideoBuffer::map(QVideoFrame::MapMode mode) +{ + MapData mapData; + + if (m_mode == QVideoFrame::NotMapped) { + CVPixelBufferLockBaseAddress(m_buffer, mode == QVideoFrame::ReadOnly + ? kCVPixelBufferLock_ReadOnly + : 0); + m_mode = mode; + } + + mapData.nPlanes = CVPixelBufferGetPlaneCount(m_buffer); + mapData.nBytes = CVPixelBufferGetDataSize(m_buffer); + Q_ASSERT(mapData.nPlanes <= 3); + + if (!mapData.nPlanes) { + // single plane + mapData.bytesPerLine[0] = CVPixelBufferGetBytesPerRow(m_buffer); + mapData.data[0] = static_cast<uchar*>(CVPixelBufferGetBaseAddress(m_buffer)); + mapData.nPlanes = mapData.data[0] ? 1 : 0; + return mapData; + } + + // For a bi-planar or tri-planar format we have to set the parameters correctly: + for (int i = 0; i < mapData.nPlanes; ++i) { + mapData.bytesPerLine[i] = CVPixelBufferGetBytesPerRowOfPlane(m_buffer, i); + mapData.data[i] = static_cast<uchar*>(CVPixelBufferGetBaseAddressOfPlane(m_buffer, i)); + } + + return mapData; +} + +void AVFVideoBuffer::unmap() +{ + if (m_mode != QVideoFrame::NotMapped) { + CVPixelBufferUnlockBaseAddress(m_buffer, m_mode == QVideoFrame::ReadOnly + ? kCVPixelBufferLock_ReadOnly + : 0); + m_mode = QVideoFrame::NotMapped; + } +} + +quint64 AVFVideoBuffer::textureHandle(int plane) const +{ +// qDebug() << "texture handle" << plane << rhi << (rhi->backend() == QRhi::Metal); + if (plane != 0) + return 0; + if (!rhi) + return 0; + if (rhi->backend() == QRhi::Metal) { + if (metalTexture == nil) { + const auto *metal = static_cast<const QRhiMetalNativeHandles *>(rhi->nativeHandles()); + + // Create a Metal Core Video texture cache from the pixel buffer. + if (CVMetalTextureCacheCreate( + kCFAllocatorDefault, + nil, + (id<MTLDevice>)metal->dev, + nil, + &cvMetalTextureCache) != kCVReturnSuccess) + qWarning() << "texture cache creation failed"; + + // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. + auto ret = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + cvMetalTextureCache, + m_buffer, nil, + MTLPixelFormatBGRA8Unorm, + CVPixelBufferGetWidth(m_buffer), CVPixelBufferGetHeight(m_buffer), + 0, + &cvMetalTexture); + if (ret != kCVReturnSuccess) + qWarning() << "texture creation failed" << ret; + metalTexture = CVMetalTextureGetTexture(cvMetalTexture); + } +// qDebug() << " -> " << quint64(metalTexture); + + // Get a Metal texture using the CoreVideo Metal texture reference. + return quint64(metalTexture); + } else if (rhi->backend() == QRhi::OpenGLES2) { + const auto *gl = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); + + auto nsGLContext = gl->context->nativeInterface<QNativeInterface::QCocoaGLContext>()->nativeContext(); + auto nsGLPixelFormat = nsGLContext.pixelFormat.CGLPixelFormatObj; + + CVReturn cvret; + // Create an OpenGL CoreVideo texture cache from the pixel buffer. + cvret = CVOpenGLTextureCacheCreate( + kCFAllocatorDefault, + nullptr, + reinterpret_cast<CGLContextObj>(nsGLContext.CGLContextObj), + nsGLPixelFormat, + nil, + &cvOpenGLTextureCache); + + // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache. + cvret = CVOpenGLTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + cvOpenGLTextureCache, + m_buffer, + nil, + &cvOpenGLTexture); + + // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image. + return CVOpenGLTextureGetName(cvOpenGLTexture); + + } + return 0; +#ifdef Q_OS_IOS + // Called from the render thread, so there is a current OpenGL context + + if (!m_renderer->m_textureCache) { + CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, + nullptr, + [EAGLContext currentContext], + nullptr, + &m_renderer->m_textureCache); + + if (err != kCVReturnSuccess) + qWarning("Error creating texture cache"); + } + + if (m_renderer->m_textureCache && !m_texture) { + CVOpenGLESTextureCacheFlush(m_renderer->m_textureCache, 0); + + CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + m_renderer->m_textureCache, + m_buffer, + nullptr, + GL_TEXTURE_2D, + GL_RGBA, + CVPixelBufferGetWidth(m_buffer), + CVPixelBufferGetHeight(m_buffer), + GL_RGBA, + GL_UNSIGNED_BYTE, + 0, + &m_texture); + if (err != kCVReturnSuccess) + qWarning("Error creating texture from buffer"); + } + + if (m_texture) + return CVOpenGLESTextureGetName(m_texture); + else + return 0; +#endif +} diff --git a/src/multimedia/platform/darwin/avfvideobuffer_p.h b/src/multimedia/platform/darwin/avfvideobuffer_p.h new file mode 100644 index 000000000..acc55f55e --- /dev/null +++ b/src/multimedia/platform/darwin/avfvideobuffer_p.h @@ -0,0 +1,99 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef AVFVIDEOBUFFER_H +#define AVFVIDEOBUFFER_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtMultimedia/qvideoframe.h> +#include <private/qabstractvideobuffer_p.h> + +#include <QtCore/qobject.h> +#include <QtCore/qmutex.h> +#include <private/avfvideosink_p.h> + +#include <CoreVideo/CVBase.h> +#include <CoreVideo/CVPixelBuffer.h> +#include <CoreVideo/CVImageBuffer.h> + +#import "Metal/Metal.h" +#import "MetalKit/MetalKit.h" + +QT_BEGIN_NAMESPACE + +struct AVFMetalTexture; +class AVFVideoBuffer : public QAbstractVideoBuffer +{ +public: + AVFVideoBuffer(QRhi *rhi, CVImageBufferRef buffer); + ~AVFVideoBuffer(); + + QVideoFrame::MapMode mapMode() const { return m_mode; } + MapData map(QVideoFrame::MapMode mode); + void unmap(); + + virtual quint64 textureHandle(int /*plane*/) const; + +private: + QRhi *rhi = nullptr; + + mutable CVMetalTextureRef cvMetalTexture = nullptr; + mutable CVMetalTextureCacheRef cvMetalTextureCache = nullptr; + mutable id<MTLTexture> metalTexture = nil; + + mutable CVOpenGLTextureRef cvOpenGLTexture = nullptr; + mutable CVOpenGLTextureCacheRef cvOpenGLTextureCache = nullptr; + + CVImageBufferRef m_buffer = nullptr; + QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/multimedia/platform/darwin/avfvideosink.mm b/src/multimedia/platform/darwin/avfvideosink.mm index 1d7147b1f..3bd88ca95 100644 --- a/src/multimedia/platform/darwin/avfvideosink.mm +++ b/src/multimedia/platform/darwin/avfvideosink.mm @@ -65,8 +65,6 @@ bool AVFVideoSink::setGraphicsType(QVideoSink::GraphicsType type) { if (type == m_graphicsType) return true; - if (type == QVideoSink::Direct3D11 || type == QVideoSink::Vulkan) - return false; m_graphicsType = type; if (m_interface) m_interface->reconfigure(); @@ -88,6 +86,13 @@ void AVFVideoSink::setWinId(WId id) m_interface->reconfigure(); } +void AVFVideoSink::setRhi(QRhi *rhi) +{ + m_rhi = rhi; + if (m_interface) + m_interface->setRhi(rhi); +} + QRect AVFVideoSink::displayRect() const { return m_displayRect; @@ -195,6 +200,8 @@ void AVFVideoSink::setLayer(CALayer *) void AVFVideoSink::setVideoSinkInterface(AVFVideoSinkInterface *interface) { m_interface = interface; + if (m_interface) + m_interface->setRhi(m_rhi); } AVFVideoSinkInterface::~AVFVideoSinkInterface() diff --git a/src/multimedia/platform/darwin/avfvideosink_p.h b/src/multimedia/platform/darwin/avfvideosink_p.h index 19cdb0b91..51bb02107 100644 --- a/src/multimedia/platform/darwin/avfvideosink_p.h +++ b/src/multimedia/platform/darwin/avfvideosink_p.h @@ -84,6 +84,8 @@ public: WId winId() const override; void setWinId(WId id) override; + void setRhi(QRhi *rhi) override; + QRect displayRect() const override; void setDisplayRect(const QRect &rect) override; @@ -118,6 +120,7 @@ private: AVFVideoSinkInterface *m_interface = nullptr; QVideoSink::GraphicsType m_graphicsType = QVideoSink::Memory; WId m_winId = 0; + QRhi *m_rhi = nullptr; NativeView *m_nativeView = nullptr; QSize m_nativeSize; @@ -140,6 +143,7 @@ public: virtual void reconfigure() = 0; virtual void updateAspectRatio() = 0; + virtual void setRhi(QRhi *) { Q_ASSERT(false); } void setLayer(CALayer *layer); diff --git a/src/multimedia/platform/darwin/camera/avfcameraimagecapture.mm b/src/multimedia/platform/darwin/camera/avfcameraimagecapture.mm index 7e88f1c71..c4404c4e8 100644 --- a/src/multimedia/platform/darwin/camera/avfcameraimagecapture.mm +++ b/src/multimedia/platform/darwin/camera/avfcameraimagecapture.mm @@ -153,7 +153,8 @@ int AVFCameraImageCapture::doCapture(const QString &actualFileName) QBuffer data(&jpgData); QImageReader reader(&data, "JPEG"); QSize size = reader.size(); - QVideoFrame frame(new QMemoryVideoBuffer(QByteArray(jpgData.constData(), jpgData.size()), -1), size, QVideoSurfaceFormat::Format_Jpeg); + QVideoFrame frame(new QMemoryVideoBuffer(QByteArray(jpgData.constData(), jpgData.size()), -1), + QVideoSurfaceFormat(size, QVideoSurfaceFormat::Format_Jpeg)); QMetaObject::invokeMethod(this, "imageAvailable", Qt::QueuedConnection, Q_ARG(int, request.captureId), Q_ARG(QVideoFrame, frame)); diff --git a/src/multimedia/platform/darwin/camera/avfcamerarenderer.mm b/src/multimedia/platform/darwin/camera/avfcamerarenderer.mm index 459dce5ca..dfff0bc82 100644 --- a/src/multimedia/platform/darwin/camera/avfcamerarenderer.mm +++ b/src/multimedia/platform/darwin/camera/avfcamerarenderer.mm @@ -44,6 +44,7 @@ #include "avfcameradebug_p.h" #include "avfcamera_p.h" #include <private/avfvideosink_p.h> +#include <private/avfvideobuffer_p.h> #include "qvideosink.h" #import <AVFoundation/AVFoundation.h> @@ -58,159 +59,6 @@ QT_USE_NAMESPACE -class CVImageVideoBuffer : public QAbstractVideoBuffer -{ -public: - CVImageVideoBuffer(CVImageBufferRef buffer, AVFCameraRenderer *renderer) -#ifndef Q_OS_IOS - : QAbstractVideoBuffer(QVideoFrame::NoHandle) -#else - : QAbstractVideoBuffer(renderer->supportsTextures() - && CVPixelBufferGetPixelFormatType(buffer) == kCVPixelFormatType_32BGRA - ? QVideoFrame::GLTextureHandle : QVideoFrame::NoHandle) - , m_texture(nullptr) - , m_renderer(renderer) -#endif - , m_buffer(buffer) - , m_mode(QVideoFrame::NotMapped) - { -#ifndef Q_OS_IOS - Q_UNUSED(renderer); -#endif // Q_OS_IOS - CVPixelBufferRetain(m_buffer); - } - - ~CVImageVideoBuffer() - { - CVImageVideoBuffer::unmap(); -#ifdef Q_OS_IOS - if (m_texture) - CFRelease(m_texture); -#endif - CVPixelBufferRelease(m_buffer); - } - - QVideoFrame::MapMode mapMode() const { return m_mode; } - - MapData map(QVideoFrame::MapMode mode) - { - MapData mapData; - - // We only support RGBA or NV12 (or Apple's version of NV12), - // they are either 0 planes or 2. - mapData.nPlanes = CVPixelBufferGetPlaneCount(m_buffer); - Q_ASSERT(mapData.nPlanes <= 2); - - if (!mapData.nPlanes) { - mapData.data[0] = map(mode, &mapData.nBytes, &mapData.bytesPerLine[0]); - mapData.nPlanes = mapData.data[0] ? 1 : 0; - return mapData; - } - - // For a bi-planar format we have to set the parameters correctly: - if (mode != QVideoFrame::NotMapped && m_mode == QVideoFrame::NotMapped) { - CVPixelBufferLockBaseAddress(m_buffer, mode == QVideoFrame::ReadOnly - ? kCVPixelBufferLock_ReadOnly - : 0); - - mapData.nBytes = CVPixelBufferGetDataSize(m_buffer); - - // At the moment we handle only bi-planar format. - mapData.bytesPerLine[0] = CVPixelBufferGetBytesPerRowOfPlane(m_buffer, 0); - mapData.bytesPerLine[1] = CVPixelBufferGetBytesPerRowOfPlane(m_buffer, 1); - - mapData.data[0] = static_cast<uchar*>(CVPixelBufferGetBaseAddressOfPlane(m_buffer, 0)); - mapData.data[1] = static_cast<uchar*>(CVPixelBufferGetBaseAddressOfPlane(m_buffer, 1)); - - m_mode = mode; - } - - return mapData; - } - - uchar *map(QVideoFrame::MapMode mode, qsizetype *numBytes, int *bytesPerLine) - { - if (mode != QVideoFrame::NotMapped && m_mode == QVideoFrame::NotMapped) { - CVPixelBufferLockBaseAddress(m_buffer, mode == QVideoFrame::ReadOnly - ? kCVPixelBufferLock_ReadOnly - : 0); - if (numBytes) - *numBytes = CVPixelBufferGetDataSize(m_buffer); - - if (bytesPerLine) - *bytesPerLine = CVPixelBufferGetBytesPerRow(m_buffer); - - m_mode = mode; - return static_cast<uchar*>(CVPixelBufferGetBaseAddress(m_buffer)); - } else { - return nullptr; - } - } - - void unmap() - { - if (m_mode != QVideoFrame::NotMapped) { - CVPixelBufferUnlockBaseAddress(m_buffer, m_mode == QVideoFrame::ReadOnly - ? kCVPixelBufferLock_ReadOnly - : 0); - m_mode = QVideoFrame::NotMapped; - } - } - - QVariant handle() const - { -#ifdef Q_OS_IOS - // Called from the render thread, so there is a current OpenGL context - - if (!m_renderer->m_textureCache) { - CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, - nullptr, - [EAGLContext currentContext], - nullptr, - &m_renderer->m_textureCache); - - if (err != kCVReturnSuccess) - qWarning("Error creating texture cache"); - } - - if (m_renderer->m_textureCache && !m_texture) { - CVOpenGLESTextureCacheFlush(m_renderer->m_textureCache, 0); - - CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, - m_renderer->m_textureCache, - m_buffer, - nullptr, - GL_TEXTURE_2D, - GL_RGBA, - CVPixelBufferGetWidth(m_buffer), - CVPixelBufferGetHeight(m_buffer), - GL_RGBA, - GL_UNSIGNED_BYTE, - 0, - &m_texture); - if (err != kCVReturnSuccess) - qWarning("Error creating texture from buffer"); - } - - if (m_texture) - return CVOpenGLESTextureGetName(m_texture); - else - return 0; -#else - return QVariant(); -#endif - } - -private: -#ifdef Q_OS_IOS - mutable CVOpenGLESTextureRef m_texture; - AVFCameraRenderer *m_renderer; -#endif - CVImageBufferRef m_buffer; - QVideoFrame::MapMode m_mode; -}; - - @interface AVFCaptureFramesDelegate : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate> - (AVFCaptureFramesDelegate *) initWithRenderer:(AVFCameraRenderer*)renderer; @@ -253,12 +101,10 @@ private: QVideoSurfaceFormat::PixelFormat format = AVFCamera::QtPixelFormatFromCVFormat(CVPixelBufferGetPixelFormatType(imageBuffer)); if (format == QVideoSurfaceFormat::Format_Invalid) -avfcamerarenderercontrol.mm return; - QVideoFrame frame(new CVImageVideoBuffer(imageBuffer, m_renderer), - QSize(width, height), - format); + QVideoFrame frame(new AVFVideoBuffer(nullptr, imageBuffer), + QVideoSurfaceFormat(QSize(width, height), format)); m_renderer->syncHandleViewfinderFrame(frame); } @@ -377,7 +223,7 @@ void AVFCameraRenderer::handleViewfinderFrame() if (m_sink && frame.isValid()) { // ### pass format to surface - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), frame.handleType()); + QVideoSurfaceFormat format = frame.surfaceFormat(); if (m_needsHorizontalMirroring) format.setMirrored(true); diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm deleted file mode 100644 index fb341ffca..000000000 --- a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer.mm +++ /dev/null @@ -1,463 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "avfvideoframerenderer_p.h" - -#include <QtOpenGL/QOpenGLFramebufferObject> -#include <QtGui/QWindow> -#include <QOpenGLShaderProgram> - -#ifdef QT_DEBUG_AVF -#include <QtCore/qdebug.h> -#endif - -#import <CoreVideo/CVBase.h> -#import <AVFoundation/AVFoundation.h> - -QT_USE_NAMESPACE - -AVFVideoFrameRenderer::AVFVideoFrameRenderer(QObject *parent) - : QObject(parent) - , m_videoLayerRenderer(nullptr) - , m_offscreenSurface(nullptr) - , m_glContext(nullptr) - , m_currentBuffer(1) - , m_isContextShared(true) -{ - m_fbo[0] = nullptr; - m_fbo[1] = nullptr; -} - -AVFVideoFrameRenderer::~AVFVideoFrameRenderer() -{ -#ifdef QT_DEBUG_AVF - qDebug() << Q_FUNC_INFO; -#endif - - [m_videoLayerRenderer release]; - [m_metalDevice release]; - delete m_fbo[0]; - delete m_fbo[1]; - delete m_offscreenSurface; - delete m_glContext; - - if (m_useCoreProfile) { - glDeleteVertexArrays(1, &m_quadVao); - glDeleteBuffers(2, m_quadVbos); - delete m_shader; - } -} - -quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) -{ - //Is layer valid - if (!layer) - return 0; - - //If the glContext isn't shared, it doesn't make sense to return a texture for us - if (m_offscreenSurface && !m_isContextShared) - return 0; - - QOpenGLFramebufferObject *fbo = initRenderer(layer); - - if (!fbo) - return 0; - - renderLayerToFBO(layer, fbo); - if (m_glContext) - m_glContext->doneCurrent(); - - return fbo->texture(); -} - -quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) -{ - m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); - - if (!m_metalDevice) - m_metalDevice = MTLCreateSystemDefaultDevice(); - - if (!m_metalTexture) { - auto desc = [[MTLTextureDescriptor alloc] init]; - desc.textureType = MTLTextureType2D; - desc.width = NSUInteger(m_targetSize.width()); - desc.height = NSUInteger(m_targetSize.height()); - desc.resourceOptions = MTLResourceStorageModePrivate; - desc.usage = MTLTextureUsageRenderTarget; - desc.pixelFormat = MTLPixelFormatRGBA8Unorm; - - m_metalTexture = [m_metalDevice newTextureWithDescriptor: desc]; - [desc release]; - } - - if (!m_videoLayerRenderer) { - m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; - [m_videoLayerRenderer retain]; - } - - if (m_videoLayerRenderer.layer != layer) { - m_videoLayerRenderer.layer = layer; - m_videoLayerRenderer.bounds = layer.bounds; - } - - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; - [m_videoLayerRenderer addUpdateRect:layer.bounds]; - [m_videoLayerRenderer render]; - [m_videoLayerRenderer endFrame]; - - return quint64(m_metalTexture); -} - -QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) -{ - //Is layer valid - if (!layer) { - return QImage(); - } - - QOpenGLFramebufferObject *fbo = initRenderer(layer); - - if (!fbo) - return QImage(); - - renderLayerToFBO(layer, fbo); - QImage fboImage = fbo->toImage(); - if (m_glContext) - m_glContext->doneCurrent(); - - return fboImage; -} - -QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *layer) -{ - - //Get size from AVPlayerLayer - m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); - - QOpenGLContext *shareContext = nullptr; -// !m_glContext && m_surface -// ? qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>()) -// : nullptr; - - //Make sure we have an OpenGL context to make current - if (shareContext || (!QOpenGLContext::currentContext() && !m_glContext)) { - - //Create Hidden QWindow surface to create context in this thread - delete m_offscreenSurface; - m_offscreenSurface = new QWindow(); - m_offscreenSurface->setSurfaceType(QWindow::OpenGLSurface); - //Needs geometry to be a valid surface, but size is not important - m_offscreenSurface->setGeometry(0, 0, 1, 1); - m_offscreenSurface->create(); - - delete m_glContext; - m_glContext = new QOpenGLContext(); - m_glContext->setFormat(m_offscreenSurface->requestedFormat()); - - if (shareContext) { - m_glContext->setShareContext(shareContext); - m_isContextShared = true; - } else { -#ifdef QT_DEBUG_AVF - qWarning("failed to get Render Thread context"); -#endif - m_isContextShared = false; - } - if (!m_glContext->create()) { - qWarning("failed to create QOpenGLContext"); - return nullptr; - } - - // CARenderer must be re-created with different current context, so release it now. - // See lines below where m_videoLayerRenderer is constructed. - if (m_videoLayerRenderer) { - [m_videoLayerRenderer release]; - m_videoLayerRenderer = nullptr; - } - - if (m_useCoreProfile) { - glDeleteVertexArrays(1, &m_quadVao); - glDeleteBuffers(2, m_quadVbos); - delete m_shader; - m_shader = nullptr; - } - } - - //Need current context - if (m_glContext) - m_glContext->makeCurrent(m_offscreenSurface); - - if (!m_metalDevice) - m_metalDevice = MTLCreateSystemDefaultDevice(); - - if (@available(macOS 10.13, *)) { - m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() == - QSurfaceFormat::CoreProfile); - } else { - m_useCoreProfile = false; - } - - // Create the CARenderer if needed for no Core OpenGL - if (!m_videoLayerRenderer) { - if (!m_useCoreProfile) { - m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() - options: nil]; - [m_videoLayerRenderer retain]; - } else if (@available(macOS 10.13, *)) { - // This is always true when m_useCoreProfile is true, but the compiler wants the check - // anyway - // Setup Core OpenGL shader, VAO, VBOs and metal renderer - m_shader = new QOpenGLShaderProgram(); - m_shader->create(); - if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core - in vec2 qt_VertexPosition; - in vec2 qt_VertexTexCoord; - out vec2 qt_TexCoord; - void main() - { - qt_TexCoord = qt_VertexTexCoord; - gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f); - })")) { - qCritical() << "Vertex shader compilation failed" << m_shader->log(); - } - if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core - in vec2 qt_TexCoord; - out vec4 fragColor; - uniform sampler2DRect videoFrame; - void main(void) - { - ivec2 textureDim = textureSize(videoFrame); - fragColor = texture(videoFrame, qt_TexCoord * textureDim); - })")) { - qCritical() << "Fragment shader compilation failed" << m_shader->log(); - } - - // Setup quad where the video frame will be attached - GLfloat vertices[] = { - -1.0f, -1.0f, - 1.0f, -1.0f, - -1.0f, 1.0f, - 1.0f, 1.0f, - }; - - GLfloat uvs[] = { - 0.0f, 0.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, - }; - - glGenVertexArrays(1, &m_quadVao); - glBindVertexArray(m_quadVao); - - // Create vertex buffer objects for vertices - glGenBuffers(2, m_quadVbos); - - // Setup vertices - glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]); - glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); - glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); - glEnableVertexAttribArray(0); - - // Setup uvs - glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]); - glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW); - glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); - glEnableVertexAttribArray(1); - - glBindVertexArray(0); - - // Setup shared Metal/OpenGL pixel buffer and textures - m_NSGLContext = QOpenGLContext::currentContext()->nativeInterface<QNativeInterface::QCocoaGLContext>()->nativeContext(); - m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj; - - NSDictionary* cvBufferProperties = @{ - static_cast<NSString*>(kCVPixelBufferOpenGLCompatibilityKey) : @YES, - static_cast<NSString*>(kCVPixelBufferMetalCompatibilityKey): @YES, - }; - - CVPixelBufferCreate(kCFAllocatorDefault, static_cast<size_t>(m_targetSize.width()), - static_cast<size_t>(m_targetSize.height()), kCVPixelFormatType_32BGRA, - static_cast<CFDictionaryRef>(cvBufferProperties), &m_CVPixelBuffer); - - m_textureName = createGLTexture(reinterpret_cast<CGLContextObj>(m_NSGLContext.CGLContextObj), - m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer, - m_CVGLTexture); - m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer, - MTLPixelFormatBGRA8Unorm, - static_cast<size_t>(m_targetSize.width()), - static_cast<size_t>(m_targetSize.height()), - m_CVMTLTexture); - - m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; - [m_videoLayerRenderer retain]; - } - } - - //Set/Change render source if needed - if (m_videoLayerRenderer.layer != layer) { - m_videoLayerRenderer.layer = layer; - m_videoLayerRenderer.bounds = layer.bounds; - } - - //Do we have FBO's already? - if ((!m_fbo[0] && !m_fbo[0]) || (m_fbo[0]->size() != m_targetSize)) { - delete m_fbo[0]; - delete m_fbo[1]; - m_fbo[0] = new QOpenGLFramebufferObject(m_targetSize); - m_fbo[1] = new QOpenGLFramebufferObject(m_targetSize); - } - - //Switch buffer target - m_currentBuffer = !m_currentBuffer; - return m_fbo[m_currentBuffer]; -} - -void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo) -{ - //Start Rendering - //NOTE: This rendering method will NOT work on iOS as there is no CARenderer in iOS - if (!fbo->bind()) { - qWarning("AVFVideoRender FBO failed to bind"); - return; - } - - glClearColor(0.0f, 0.0f, 0.0f, 1.0f); - glClear(GL_COLOR_BUFFER_BIT); - - glViewport(0, 0, m_targetSize.width(), m_targetSize.height()); - - if (m_useCoreProfile) { - CGLLockContext(m_NSGLContext.CGLContextObj); - m_shader->bind(); - glBindVertexArray(m_quadVao); - } else { - glMatrixMode(GL_PROJECTION); - glPushMatrix(); - glLoadIdentity(); - - // Render to FBO with inverted Y - glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); - - glMatrixMode(GL_MODELVIEW); - glPushMatrix(); - glLoadIdentity(); - } - - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr]; - [m_videoLayerRenderer addUpdateRect:layer.bounds]; - [m_videoLayerRenderer render]; - [m_videoLayerRenderer endFrame]; - - if (m_useCoreProfile) { - glActiveTexture(0); - glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName); - - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - glBindTexture(GL_TEXTURE_RECTANGLE, 0); - - glBindVertexArray(0); - - m_shader->release(); - - CGLFlushDrawable(m_NSGLContext.CGLContextObj); - CGLUnlockContext(m_NSGLContext.CGLContextObj); - } else { - glMatrixMode(GL_MODELVIEW); - glPopMatrix(); - glMatrixMode(GL_PROJECTION); - glPopMatrix(); - } - - glFinish(); //Rendering needs to be done before passing texture to video frame - - fbo->release(); -} - -GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache, - CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef) -{ - CVReturn cvret; - // Create an OpenGL CoreVideo texture cache from the pixel buffer. - cvret = CVOpenGLTextureCacheCreate( - kCFAllocatorDefault, - nil, - cglContextObj, - cglPixelFormtObj, - nil, - &cvglTextureCache); - - // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache. - cvret = CVOpenGLTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - cvglTextureCache, - cvPixelBufferRef, - nil, - &cvOpenGLTextureRef); - - // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image. - return CVOpenGLTextureGetName(cvOpenGLTextureRef); -} - -id<MTLTexture> AVFVideoFrameRenderer::createMetalTexture(id<MTLDevice> mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef, - MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef) -{ - CVReturn cvret; - // Create a Metal Core Video texture cache from the pixel buffer. - cvret = CVMetalTextureCacheCreate( - kCFAllocatorDefault, - nil, - mtlDevice, - nil, - &cvMetalTextureCacheRef); - - // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. - cvret = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - cvMetalTextureCacheRef, - cvPixelBufferRef, nil, - pixelFormat, - width, height, - 0, - &cvMetalTextureRef); - - // Get a Metal texture using the CoreVideo Metal texture reference. - return CVMetalTextureGetTexture(cvMetalTextureRef); -} diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_ios.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_ios.mm deleted file mode 100644 index 7f61249a1..000000000 --- a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_ios.mm +++ /dev/null @@ -1,300 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "private/avfvideoframerenderer_ios_p.h" - -#include <QtOpenGL/QOpenGLFramebufferObject> -#include <QtOpenGL/QOpenGLShaderProgram> -#include <QtGui/QOffscreenSurface> - -#ifdef QT_DEBUG_AVF -#include <QtCore/qdebug.h> -#endif - -#import <CoreVideo/CVBase.h> -#import <AVFoundation/AVFoundation.h> -QT_USE_NAMESPACE - -AVFVideoFrameRenderer::AVFVideoFrameRenderer(QObject *parent) - : QObject(parent) -{ -} - -AVFVideoFrameRenderer::~AVFVideoFrameRenderer() -{ -#ifdef QT_DEBUG_AVF - qDebug() << Q_FUNC_INFO; -#endif - - [m_videoOutput release]; // sending to nil is fine - if (m_textureCache) - CFRelease(m_textureCache); - if (m_metalTextureCache) - CFRelease(m_metalTextureCache); - [m_metalDevice release]; - delete m_offscreenSurface; - delete m_glContext; -} - -void AVFVideoFrameRenderer::setPlayerLayer(AVPlayerLayer *layer) -{ - Q_UNUSED(layer); - if (m_videoOutput) { - [m_videoOutput release]; - m_videoOutput = nullptr; - // will be re-created in first call to copyPixelBufferFromLayer - } -} - -quint64 AVFVideoFrameRenderer::renderLayerToMTLTexture(AVPlayerLayer *layer) -{ - if (!m_metalDevice) - m_metalDevice = MTLCreateSystemDefaultDevice(); - - if (!m_metalTextureCache) { - CVReturn err = CVMetalTextureCacheCreate(kCFAllocatorDefault, nullptr, - m_metalDevice, nullptr, &m_metalTextureCache); - if (err) { - qWarning() << "Error at CVMetalTextureCacheCreate" << err; - return 0; - } - } - - size_t width = 0, height = 0; - CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); - - if (!pixelBuffer) - return 0; - - CVMetalTextureCacheFlush(m_metalTextureCache, 0); - - CVMetalTextureRef texture = nil; - CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_metalTextureCache, pixelBuffer, NULL, - MTLPixelFormatBGRA8Unorm_sRGB, width, height, 0, &texture); - - if (!texture || err) - qWarning("CVMetalTextureCacheCreateTextureFromImage failed (error: %d)", err); - - CVPixelBufferRelease(pixelBuffer); - quint64 tex = 0; - if (texture) { - tex = quint64(CVMetalTextureGetTexture(texture)); - CFRelease(texture); - } - - return tex; -} - -quint64 AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) -{ - initRenderer(); - - // If the glContext isn't shared, it doesn't make sense to return a texture for us - if (!m_isContextShared) - return 0; - - size_t dummyWidth = 0, dummyHeight = 0; - auto texture = createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); - auto tex = quint64(CVOGLTextureGetName(texture)); - CFRelease(texture); - - return tex; -} - -static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; -static NSNumber* const AVF_PIXEL_FORMAT_VALUE = [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]; -static NSDictionary* const AVF_OUTPUT_SETTINGS = [NSDictionary dictionaryWithObject:AVF_PIXEL_FORMAT_VALUE forKey:AVF_PIXEL_FORMAT_KEY]; - - -CVPixelBufferRef AVFVideoFrameRenderer::copyPixelBufferFromLayer(AVPlayerLayer *layer, - size_t& width, size_t& height) -{ - //Is layer valid - if (!layer) { -#ifdef QT_DEBUG_AVF - qWarning("copyPixelBufferFromLayer: invalid layer"); -#endif - return nullptr; - } - - if (!m_videoOutput) { - m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:AVF_OUTPUT_SETTINGS]; - [m_videoOutput setDelegate:nil queue:nil]; - AVPlayerItem * item = [[layer player] currentItem]; - [item addOutput:m_videoOutput]; - } - - CFTimeInterval currentCAFrameTime = CACurrentMediaTime(); - CMTime currentCMFrameTime = [m_videoOutput itemTimeForHostTime:currentCAFrameTime]; - // happens when buffering / loading - if (CMTimeCompare(currentCMFrameTime, kCMTimeZero) < 0) { - return nullptr; - } - - CVPixelBufferRef pixelBuffer = [m_videoOutput copyPixelBufferForItemTime:currentCMFrameTime - itemTimeForDisplay:nil]; - if (!pixelBuffer) { -#ifdef QT_DEBUG_AVF - qWarning("copyPixelBufferForItemTime returned nil"); - CMTimeShow(currentCMFrameTime); -#endif - return nullptr; - } - - width = CVPixelBufferGetWidth(pixelBuffer); - height = CVPixelBufferGetHeight(pixelBuffer); - return pixelBuffer; -} - -CVOGLTextureRef AVFVideoFrameRenderer::createCacheTextureFromLayer(AVPlayerLayer *layer, - size_t& width, size_t& height) -{ - CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); - - if (!pixelBuffer) - return nullptr; - - CVOGLTextureCacheFlush(m_textureCache, 0); - - CVOGLTextureRef texture = nullptr; - CVReturn err = CVOGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_textureCache, pixelBuffer, nullptr, - GL_TEXTURE_2D, GL_RGBA, - (GLsizei) width, (GLsizei) height, - GL_BGRA, GL_UNSIGNED_BYTE, 0, - &texture); - - if (!texture || err) { -#ifdef QT_DEBUG_AVF - qWarning("CVOGLTextureCacheCreateTextureFromImage failed (error: %d)", err); -#endif - } - - CVPixelBufferRelease(pixelBuffer); - - return texture; -} - -QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) -{ - size_t width = 0; - size_t height = 0; - CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); - - if (!pixelBuffer) - return QImage(); - - OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); - if (pixelFormat != kCVPixelFormatType_32BGRA) { -#ifdef QT_DEBUG_AVF - qWarning("CVPixelBuffer format is not BGRA32 (got: %d)", static_cast<quint32>(pixelFormat)); -#endif - return QImage(); - } - - CVPixelBufferLockBaseAddress(pixelBuffer, 0); - char *data = (char *)CVPixelBufferGetBaseAddress(pixelBuffer); - size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer); - - // format here is not relevant, only using for storage - QImage img = QImage(width, height, QImage::Format_ARGB32); - for (size_t j = 0; j < height; j++) { - memcpy(img.scanLine(j), data, width * 4); - data += stride; - } - - CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); - CVPixelBufferRelease(pixelBuffer); - return img; -} - -void AVFVideoFrameRenderer::initRenderer() -{ - // even for using a texture directly, we need to be able to make a context current, - // so we need an offscreen, and we shouldn't assume we can make the surface context - // current on that offscreen, so use our own (sharing with it). Slightly - // excessive but no performance penalty and makes the QImage path easier to maintain - - //Make sure we have an OpenGL context to make current - if (!m_glContext) { - //Create OpenGL context and set share context from surface - QOpenGLContext *shareContext = nullptr; -// if (m_surface) { -// shareContext = qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>()); -// } - - m_glContext = new QOpenGLContext(); - if (shareContext) { - m_glContext->setShareContext(shareContext); - m_isContextShared = true; - } else { -#ifdef QT_DEBUG_AVF - qWarning("failed to get Render Thread context"); -#endif - m_isContextShared = false; - } - if (!m_glContext->create()) { -#ifdef QT_DEBUG_AVF - qWarning("failed to create QOpenGLContext"); -#endif - return; - } - } - - if (!m_offscreenSurface) { - m_offscreenSurface = new QOffscreenSurface(); - m_offscreenSurface->setFormat(m_glContext->format()); - m_offscreenSurface->create(); - } - - //Need current context - m_glContext->makeCurrent(m_offscreenSurface); - - if (!m_textureCache) { - // Create a new open gl texture cache - CVReturn err = CVOGLTextureCacheCreate(kCFAllocatorDefault, nullptr, - [EAGLContext currentContext], - nullptr, &m_textureCache); - if (err) { - #ifdef QT_DEBUG_AVF - qWarning("Error at CVOGLTextureCacheCreate %d", err); - #endif - } - } - -} diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_ios_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_ios_p.h deleted file mode 100644 index 0d816282b..000000000 --- a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_ios_p.h +++ /dev/null @@ -1,129 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef AVFVIDEOFRAMERENDERER_H -#define AVFVIDEOFRAMERENDERER_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <QtCore/QObject> -#include <QtGui/QImage> -#include <QtGui/QOpenGLContext> -#include <QtCore/QSize> - -#import "Metal/Metal.h" -#import "MetalKit/MetalKit.h" - -@class AVPlayerLayer; -@class AVPlayerItemVideoOutput; - -QT_BEGIN_NAMESPACE - -class QOpenGLContext; -class QOpenGLFramebufferObject; -class QOpenGLShaderProgram; -class QOffscreenSurface; - -typedef struct __CVBuffer *CVBufferRef; -typedef CVBufferRef CVImageBufferRef; -typedef CVImageBufferRef CVPixelBufferRef; -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -typedef struct __CVOpenGLESTextureCache *CVOpenGLESTextureCacheRef; -typedef CVImageBufferRef CVOpenGLESTextureRef; -// helpers to avoid boring if def -typedef CVOpenGLESTextureCacheRef CVOGLTextureCacheRef; -typedef CVOpenGLESTextureRef CVOGLTextureRef; -#define CVOGLTextureGetTarget CVOpenGLESTextureGetTarget -#define CVOGLTextureGetName CVOpenGLESTextureGetName -#define CVOGLTextureCacheCreate CVOpenGLESTextureCacheCreate -#define CVOGLTextureCacheCreateTextureFromImage CVOpenGLESTextureCacheCreateTextureFromImage -#define CVOGLTextureCacheFlush CVOpenGLESTextureCacheFlush -#else -typedef struct __CVOpenGLTextureCache *CVOpenGLTextureCacheRef; -typedef CVImageBufferRef CVOpenGLTextureRef; -// helpers to avoid boring if def -typedef CVOpenGLTextureCacheRef CVOGLTextureCacheRef; -typedef CVOpenGLTextureRef CVOGLTextureRef; -#define CVOGLTextureGetTarget CVOpenGLTextureGetTarget -#define CVOGLTextureGetName CVOpenGLTextureGetName -#define CVOGLTextureCacheCreate CVOpenGLTextureCacheCreate -#define CVOGLTextureCacheCreateTextureFromImage CVOpenGLTextureCacheCreateTextureFromImage -#define CVOGLTextureCacheFlush CVOpenGLTextureCacheFlush -#endif - -class AVFVideoFrameRenderer : public QObject -{ -public: - AVFVideoFrameRenderer(QObject *parent = nullptr); - - virtual ~AVFVideoFrameRenderer(); - - void setPlayerLayer(AVPlayerLayer *layer); - - quint64 renderLayerToTexture(AVPlayerLayer *layer); - quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); - QImage renderLayerToImage(AVPlayerLayer *layer); - -private: - void initRenderer(); - CVPixelBufferRef copyPixelBufferFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); - CVOGLTextureRef createCacheTextureFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); - - QOpenGLContext *m_glContext = nullptr; - QOffscreenSurface *m_offscreenSurface = nullptr; - CVOGLTextureCacheRef m_textureCache = nullptr; - AVPlayerItemVideoOutput* m_videoOutput = nil; - bool m_isContextShared = true; - - id<MTLDevice> m_metalDevice = nil; - CVMetalTextureCacheRef m_metalTextureCache = nil; -}; - -QT_END_NAMESPACE - -#endif // AVFVIDEOFRAMERENDERER_H diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h deleted file mode 100644 index 2ea0a0b6f..000000000 --- a/src/multimedia/platform/darwin/mediaplayer/avfvideoframerenderer_p.h +++ /dev/null @@ -1,135 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef AVFVIDEOFRAMERENDERER_H -#define AVFVIDEOFRAMERENDERER_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <QtCore/QObject> -#include <QtGui/QImage> -#include <QtGui/QOpenGLContext> -#include <QtCore/QSize> - -#import "Metal/Metal.h" -#import "MetalKit/MetalKit.h" - -@class CARenderer; -@class AVPlayerLayer; - -QT_BEGIN_NAMESPACE - -class QOpenGLFramebufferObject; -class QOpenGLShaderProgram; -class QWindow; -class QOpenGLContext; - -class AVFVideoFrameRenderer : public QObject -{ -public: - AVFVideoFrameRenderer(QObject *parent = nullptr); - - virtual ~AVFVideoFrameRenderer(); - - quint64 renderLayerToTexture(AVPlayerLayer *layer); - quint64 renderLayerToMTLTexture(AVPlayerLayer *layer); - QImage renderLayerToImage(AVPlayerLayer *layer); - - static GLuint createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, - CVOpenGLTextureCacheRef cvglTextureCache, - CVPixelBufferRef cvPixelBufferRef, - CVOpenGLTextureRef cvOpenGLTextureRef); - - static id<MTLTexture> createMetalTexture(id<MTLDevice> mtlDevice, - CVMetalTextureCacheRef cvMetalTextureCacheRef, - CVPixelBufferRef cvPixelBufferRef, - MTLPixelFormat pixelFormat, size_t width, size_t height, - CVMetalTextureRef cvMetalTextureRef); - -private: - QOpenGLFramebufferObject* initRenderer(AVPlayerLayer *layer); - void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); - void renderLayerToFBOCoreOpenGL(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); - - CARenderer *m_videoLayerRenderer; - QOpenGLFramebufferObject *m_fbo[2]; - QOpenGLShaderProgram *m_shader = nullptr; - QWindow *m_offscreenSurface; - QOpenGLContext *m_glContext; - QSize m_targetSize; - - bool m_useCoreProfile = false; - - // Shared pixel buffer - CVPixelBufferRef m_CVPixelBuffer; - - // OpenGL Texture - CVOpenGLTextureCacheRef m_CVGLTextureCache; - CVOpenGLTextureRef m_CVGLTexture; - CGLPixelFormatObj m_CGLPixelFormat; - GLuint m_textureName = 0; - - // Metal Texture - CVMetalTextureRef m_CVMTLTexture; - CVMetalTextureCacheRef m_CVMTLTextureCache; - - NSOpenGLContext *m_NSGLContext = nullptr; - - GLuint m_quadVao = 0; - GLuint m_quadVbos[2]; - - uint m_currentBuffer; - bool m_isContextShared; - - id<MTLDevice> m_metalDevice = nil; - id<MTLTexture> m_metalTexture = nil; -}; - -QT_END_NAMESPACE - -#endif // AVFVIDEOFRAMERENDERER_H diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm index 75e687b81..293ad519e 100644 --- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm +++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm @@ -39,50 +39,23 @@ #include "avfvideorenderercontrol_p.h" #include "avfdisplaylink_p.h" - -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -#include "avfvideoframerenderer_ios_p.h" -#else -#include "avfvideoframerenderer_p.h" -#endif +#include <private/avfvideobuffer_p.h> #include <private/qabstractvideobuffer_p.h> #include <QtMultimedia/qvideosurfaceformat.h> #include <private/qimagevideobuffer_p.h> #include <private/avfvideosink_p.h> +#include <QtGui/private/qrhi_p.h> #include <QtCore/qdebug.h> #import <AVFoundation/AVFoundation.h> +#include <CoreVideo/CVPixelBuffer.h> +#include <CoreVideo/CVImageBuffer.h> QT_USE_NAMESPACE -class TextureVideoBuffer : public QAbstractVideoBuffer -{ -public: - TextureVideoBuffer(QVideoFrame::HandleType type, quint64 tex) - : QAbstractVideoBuffer(type) - , m_texture(tex) - {} - - virtual ~TextureVideoBuffer() - { - } - - QVideoFrame::MapMode mapMode() const override { return QVideoFrame::NotMapped; } - MapData map(QVideoFrame::MapMode /*mode*/) override { return {}; } - void unmap() override {} - - QVariant handle() const override - { - return QVariant::fromValue<unsigned long long>(m_texture); - } - -private: - quint64 m_texture; -}; - AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QObject(parent) { @@ -113,14 +86,8 @@ void AVFVideoRendererControl::reconfigure() renderToNativeView(shouldRenderToWindow()); if (rendersToWindow()) { - if (m_frameRenderer) { - delete m_frameRenderer; - m_frameRenderer = nullptr; - } m_displayLink->stop(); } else { - if (!m_frameRenderer) - m_frameRenderer = new AVFVideoFrameRenderer(this); #if defined(Q_OS_IOS) || defined(Q_OS_TVOS) if (m_layer) m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(m_layer)); @@ -148,61 +115,75 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) } auto *layer = playerLayer(); - if (!layer.readyForDisplay || !m_frameRenderer) + if (!layer.readyForDisplay) return; nativeSizeChanged(); QVideoFrame frame; - if (type == QVideoSink::Metal || type == QVideoSink::NativeTexture) { - quint64 tex = m_frameRenderer->renderLayerToMTLTexture(layer); - if (tex == 0) - return; + size_t width, height; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(width, height); + if (!pixelBuffer) + return; + AVFVideoBuffer *buffer = new AVFVideoBuffer(m_rhi, pixelBuffer); + CVPixelBufferRelease(pixelBuffer); - auto buffer = new TextureVideoBuffer(QVideoFrame::MTLTextureHandle, tex); - frame = QVideoFrame(buffer, nativeSize(), QVideoSurfaceFormat::Format_BGR32); - if (!frame.isValid()) - return; + QVideoSurfaceFormat format(QSize(width, height), QVideoSurfaceFormat::Format_ARGB32); - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::MTLTextureHandle); -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); -#else - format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); -#endif - // #### QVideoFrame needs the surfaceformat! - } else if (type == QVideoSink::OpenGL) { - quint64 tex = m_frameRenderer->renderLayerToTexture(layer); - //Make sure we got a valid texture - if (tex == 0) - return; - - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(QVideoFrame::GLTextureHandle, tex); - frame = QVideoFrame(buffer, nativeSize(), QVideoSurfaceFormat::Format_BGR32); - if (!frame.isValid()) - return; - - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::GLTextureHandle); -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); -#else - format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); + frame = QVideoFrame(buffer, format); + m_sink->videoSink()->newVideoFrame(frame); +} + +// ### Should probably ask for a YUV format instead +static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; +static NSNumber* const AVF_PIXEL_FORMAT_VALUE = [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]; +static NSDictionary* const AVF_OUTPUT_SETTINGS = [NSDictionary dictionaryWithObject:AVF_PIXEL_FORMAT_VALUE forKey:AVF_PIXEL_FORMAT_KEY]; + +CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width, size_t& height) +{ + AVPlayerLayer *layer = playerLayer(); + //Is layer valid + if (!layer) { +#ifdef QT_DEBUG_AVF + qWarning("copyPixelBufferFromLayer: invalid layer"); #endif - } else { - Q_ASSERT(type == QVideoSink::Memory); - //fallback to rendering frames to QImages - QImage frameData = m_frameRenderer->renderLayerToImage(layer); + return nullptr; + } - if (frameData.isNull()) - return; + if (!m_videoOutput) { + m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:AVF_OUTPUT_SETTINGS]; + [m_videoOutput setDelegate:nil queue:nil]; + AVPlayerItem * item = [[layer player] currentItem]; + [item addOutput:m_videoOutput]; + } - QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData); - frame = QVideoFrame(buffer, nativeSize(), QVideoSurfaceFormat::Format_ARGB32_Premultiplied); - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QVideoFrame::NoHandle); + CFTimeInterval currentCAFrameTime = CACurrentMediaTime(); + CMTime currentCMFrameTime = [m_videoOutput itemTimeForHostTime:currentCAFrameTime]; + // happens when buffering / loading + if (CMTimeCompare(currentCMFrameTime, kCMTimeZero) < 0) { + return nullptr; } - m_sink->videoSink()->newVideoFrame(frame); + CVPixelBufferRef pixelBuffer = [m_videoOutput copyPixelBufferForItemTime:currentCMFrameTime + itemTimeForDisplay:nil]; + if (!pixelBuffer) { +#ifdef QT_DEBUG_AVF + qWarning("copyPixelBufferForItemTime returned nil"); + CMTimeShow(currentCMFrameTime); +#endif + return nullptr; + } + + width = CVPixelBufferGetWidth(pixelBuffer); + height = CVPixelBufferGetHeight(pixelBuffer); +// auto f = CVPixelBufferGetPixelFormatType(pixelBuffer); +// char fmt[5]; +// memcpy(fmt, &f, 4); +// fmt[4] = 0; +// qDebug() << "copyPixelBuffer" << f << fmt << width << height; + return pixelBuffer; } + void AVFVideoRendererControl::updateAspectRatio() { if (!m_layer) @@ -226,4 +207,9 @@ void AVFVideoRendererControl::updateAspectRatio() playerLayer().videoGravity = gravity; } +void AVFVideoRendererControl::setRhi(QRhi *rhi) +{ + m_rhi = rhi; +} + #include "moc_avfvideorenderercontrol_p.cpp" diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h index 41e5b167b..7f4086a6d 100644 --- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h +++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol_p.h @@ -58,13 +58,14 @@ #include <private/avfvideosink_p.h> #include <CoreVideo/CVBase.h> +#include <CoreVideo/CVPixelBuffer.h> Q_FORWARD_DECLARE_OBJC_CLASS(CALayer); +Q_FORWARD_DECLARE_OBJC_CLASS(AVPlayerItemVideoOutput); QT_BEGIN_NAMESPACE class AVFDisplayLink; -class AVFVideoFrameRenderer; class AVFVideoRendererControl : public QObject, public AVFVideoSinkInterface { @@ -76,17 +77,19 @@ public: // AVFVideoSinkInterface void reconfigure() override; void updateAspectRatio() override; + void setRhi(QRhi *rhi) override; private Q_SLOTS: void updateVideoFrame(const CVTimeStamp &ts); private: AVPlayerLayer *playerLayer() const { return static_cast<AVPlayerLayer *>(m_layer); } + CVPixelBufferRef copyPixelBufferFromLayer(size_t& width, size_t& height); QMutex m_mutex; - - AVFVideoFrameRenderer *m_frameRenderer = nullptr; + QRhi *m_rhi = nullptr; AVFDisplayLink *m_displayLink = nullptr; + AVPlayerItemVideoOutput *m_videoOutput = nullptr; }; QT_END_NAMESPACE |