diff options
Diffstat (limited to 'src')
42 files changed, 774 insertions, 935 deletions
diff --git a/src/gsttools/qgstreamerplayersession.cpp b/src/gsttools/qgstreamerplayersession.cpp index adf11b022..d36f73b2a 100644 --- a/src/gsttools/qgstreamerplayersession.cpp +++ b/src/gsttools/qgstreamerplayersession.cpp @@ -1587,7 +1587,7 @@ void QGstreamerPlayerSession::updateVideoResolutionTag() void QGstreamerPlayerSession::updateDuration() { gint64 gstDuration = 0; - int duration = 0; + qint64 duration = 0; if (m_pipeline && qt_gst_element_query_duration(m_pipeline, GST_FORMAT_TIME, &gstDuration)) duration = gstDuration / 1000000; diff --git a/src/multimedia/audio/qaudiohelpers.cpp b/src/multimedia/audio/qaudiohelpers.cpp index fae591477..1cd189d6e 100644 --- a/src/multimedia/audio/qaudiohelpers.cpp +++ b/src/multimedia/audio/qaudiohelpers.cpp @@ -103,19 +103,19 @@ template<class T> struct signedVersion {}; template<> struct signedVersion<quint8> { typedef qint8 TS; - enum {offset = 0x80}; + static constexpr int offset = 0x80; }; template<> struct signedVersion<quint16> { typedef qint16 TS; - enum {offset = 0x8000}; + static constexpr int offset = 0x8000; }; template<> struct signedVersion<quint32> { typedef qint32 TS; - enum {offset = 0x80000000}; + static constexpr uint offset = 0x80000000; }; template<class T> void adjustUnsignedSamples(qreal factor, const void *src, void *dst, int samples) diff --git a/src/multimedia/audio/qsoundeffect_pulse_p.cpp b/src/multimedia/audio/qsoundeffect_pulse_p.cpp index 51cdde026..a141f7e36 100644 --- a/src/multimedia/audio/qsoundeffect_pulse_p.cpp +++ b/src/multimedia/audio/qsoundeffect_pulse_p.cpp @@ -552,8 +552,10 @@ void QSoundEffectPrivate::setVolume(qreal volume) m_volume = qBound(qreal(0), volume, qreal(1)); locker.unlock(); - if (!m_playing && m_pulseStream) + if (!m_playing && m_pulseStream) { + PulseDaemonLocker locker; pa_stream_flush(m_pulseStream, volume_stream_flush_callback, m_ref->getRef()); + } emit volumeChanged(); } diff --git a/src/multimedia/video/qabstractvideobuffer.cpp b/src/multimedia/video/qabstractvideobuffer.cpp index f0dd6d2eb..8762150d0 100644 --- a/src/multimedia/video/qabstractvideobuffer.cpp +++ b/src/multimedia/video/qabstractvideobuffer.cpp @@ -96,12 +96,15 @@ int QAbstractVideoBufferPrivate::map( Identifies the type of a video buffers handle. \value NoHandle The buffer has no handle, its data can only be accessed by mapping the buffer. - \value GLTextureHandle The handle of the buffer is an OpenGL texture ID. + \value GLTextureHandle The handle of the buffer is an OpenGL texture ID + of an undefined and platform dependent target type. \value XvShmImageHandle The handle contains pointer to shared memory XVideo image. \value CoreImageHandle The handle contains pointer to \macos CIImage. \value QPixmapHandle The handle of the buffer is a QPixmap. \value EGLImageHandle The handle of the buffer is an EGLImageKHR. \value UserHandle Start value for user defined handle types. + \value GLTextureRectangleHandle The handle of the buffer is an OpenGL texture ID + of target type \c GL_TEXTURE_RECTANGLE. \sa handleType() */ diff --git a/src/multimedia/video/qabstractvideobuffer.h b/src/multimedia/video/qabstractvideobuffer.h index 2352c0f3d..a3afc18e4 100644 --- a/src/multimedia/video/qabstractvideobuffer.h +++ b/src/multimedia/video/qabstractvideobuffer.h @@ -64,6 +64,7 @@ public: CoreImageHandle, QPixmapHandle, EGLImageHandle, + GLTextureRectangleHandle, UserHandle = 1000 }; diff --git a/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp b/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp index 7cc3ad619..c0484a139 100644 --- a/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp +++ b/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp @@ -188,19 +188,30 @@ void QAndroidCaptureSession::setState(QMediaRecorder::State state) start(); break; case QMediaRecorder::PausedState: - // Not supported by Android API - qWarning("QMediaRecorder::PausedState is not supported on Android"); + pause(); break; } } void QAndroidCaptureSession::start() { - if (m_state == QMediaRecorder::RecordingState || m_status != QMediaRecorder::LoadedStatus) + if (m_state == QMediaRecorder::RecordingState + || (m_status != QMediaRecorder::LoadedStatus && m_status != QMediaRecorder::PausedStatus)) return; setStatus(QMediaRecorder::StartingStatus); + if (m_state == QMediaRecorder::PausedState) { + if (!m_mediaRecorder || !m_mediaRecorder->resume()) { + emit error(QMediaRecorder::FormatError, QLatin1String("Unable to resume the media recorder.")); + if (m_cameraSession) + restartViewfinder(); + } else { + updateStartState(); + } + return; + } + if (m_mediaRecorder) { m_mediaRecorder->release(); delete m_mediaRecorder; @@ -289,7 +300,11 @@ void QAndroidCaptureSession::start() restartViewfinder(); return; } + updateStartState(); +} +void QAndroidCaptureSession::updateStartState() +{ m_elapsedTime.start(); m_notifyTimer.start(); updateDuration(); @@ -302,9 +317,20 @@ void QAndroidCaptureSession::start() m_cameraSession->camera()->setupPreviewFrameCallback(); } + QMediaRecorder::State oldState = m_state; + QMediaRecorder::Status oldStatus = m_status; + m_state = QMediaRecorder::RecordingState; - emit stateChanged(m_state); - setStatus(QMediaRecorder::RecordingStatus); + m_status = QMediaRecorder::RecordingStatus; + + m_actualOutputLocation = m_usedOutputLocation; + emit actualLocationChanged(m_actualOutputLocation); + + if (m_state != oldState) + emit stateChanged(m_state); + + if (m_status != oldStatus) + emit statusChanged(m_status); } void QAndroidCaptureSession::stop(bool error) @@ -317,6 +343,7 @@ void QAndroidCaptureSession::stop(bool error) m_mediaRecorder->stop(); m_notifyTimer.stop(); updateDuration(); + m_previousElapsedTime = 0; m_elapsedTime.invalidate(); m_mediaRecorder->release(); delete m_mediaRecorder; @@ -347,6 +374,23 @@ void QAndroidCaptureSession::stop(bool error) setStatus(QMediaRecorder::LoadedStatus); } +void QAndroidCaptureSession::pause() +{ + if (m_state == QMediaRecorder::PausedState || m_mediaRecorder == 0) + return; + + setStatus(QMediaRecorder::PausedStatus); + + m_mediaRecorder->pause(); + m_notifyTimer.stop(); + updateDuration(); + m_previousElapsedTime = m_duration; + m_elapsedTime.invalidate(); + + m_state = QMediaRecorder::PausedState; + emit stateChanged(m_state); +} + void QAndroidCaptureSession::setStatus(QMediaRecorder::Status status) { if (m_status == status) @@ -503,7 +547,7 @@ void QAndroidCaptureSession::restartViewfinder() void QAndroidCaptureSession::updateDuration() { if (m_elapsedTime.isValid()) - m_duration = m_elapsedTime.elapsed(); + m_duration = m_elapsedTime.elapsed() + m_previousElapsedTime; emit durationChanged(m_duration); } diff --git a/src/plugins/android/src/mediacapture/qandroidcapturesession.h b/src/plugins/android/src/mediacapture/qandroidcapturesession.h index 8cfb9ad2a..cee148a4c 100644 --- a/src/plugins/android/src/mediacapture/qandroidcapturesession.h +++ b/src/plugins/android/src/mediacapture/qandroidcapturesession.h @@ -136,7 +136,9 @@ private: CaptureProfile getProfile(int id); void start(); + void updateStartState(); void stop(bool error = false); + void pause(); void setStatus(QMediaRecorder::Status status); @@ -154,6 +156,7 @@ private: QElapsedTimer m_elapsedTime; QTimer m_notifyTimer; qint64 m_duration; + qint64 m_previousElapsedTime = 0; QMediaRecorder::State m_state; QMediaRecorder::Status m_status; diff --git a/src/plugins/android/src/wrappers/jni/androidcamera.cpp b/src/plugins/android/src/wrappers/jni/androidcamera.cpp index 3ea7bc773..50baaed21 100644 --- a/src/plugins/android/src/wrappers/jni/androidcamera.cpp +++ b/src/plugins/android/src/wrappers/jni/androidcamera.cpp @@ -797,6 +797,12 @@ void AndroidCamera::getCameraInfo(int id, AndroidCameraInfo *info) default: break; } + // Add a number to allow correct access to cameras on systems with two + // (and more) front/back cameras + if (id > 1) { + info->name.append(QByteArray::number(id)); + info->description.append(QString(" %1").arg(id)); + } } void AndroidCamera::startPreview() diff --git a/src/plugins/android/src/wrappers/jni/androidmediarecorder.cpp b/src/plugins/android/src/wrappers/jni/androidmediarecorder.cpp index d607ab806..10508a358 100644 --- a/src/plugins/android/src/wrappers/jni/androidmediarecorder.cpp +++ b/src/plugins/android/src/wrappers/jni/androidmediarecorder.cpp @@ -213,6 +213,32 @@ void AndroidMediaRecorder::stop() } } +void AndroidMediaRecorder::pause() +{ + QJNIEnvironmentPrivate env; + m_mediaRecorder.callMethod<void>("pause"); + if (env->ExceptionCheck()) { +#ifdef QT_DEBUG + env->ExceptionDescribe(); +#endif + env->ExceptionClear(); + } +} + +bool AndroidMediaRecorder::resume() +{ + QJNIEnvironmentPrivate env; + m_mediaRecorder.callMethod<void>("resume"); + if (env->ExceptionCheck()) { +#ifdef QT_DEBUG + env->ExceptionDescribe(); +#endif + env->ExceptionClear(); + return false; + } + return true; +} + void AndroidMediaRecorder::setAudioChannels(int numChannels) { m_mediaRecorder.callMethod<void>("setAudioChannels", "(I)V", numChannels); diff --git a/src/plugins/android/src/wrappers/jni/androidmediarecorder.h b/src/plugins/android/src/wrappers/jni/androidmediarecorder.h index e4b3a80ea..55b370cf1 100644 --- a/src/plugins/android/src/wrappers/jni/androidmediarecorder.h +++ b/src/plugins/android/src/wrappers/jni/androidmediarecorder.h @@ -138,6 +138,8 @@ public: bool start(); void stop(); + void pause(); + bool resume(); void setAudioChannels(int numChannels); void setAudioEncoder(AudioEncoder encoder); diff --git a/src/plugins/avfoundation/camera/avfcameraflashcontrol.mm b/src/plugins/avfoundation/camera/avfcameraflashcontrol.mm index 42303ce17..1e27dc472 100644 --- a/src/plugins/avfoundation/camera/avfcameraflashcontrol.mm +++ b/src/plugins/avfoundation/camera/avfcameraflashcontrol.mm @@ -171,6 +171,20 @@ bool AVFCameraFlashControl::applyFlashSettings() return false; } + auto setAvTorchModeSafe = [&captureDevice](AVCaptureTorchMode avTorchMode) { + if ([captureDevice isTorchModeSupported:avTorchMode]) + captureDevice.torchMode = avTorchMode; + else + qDebugCamera() << Q_FUNC_INFO << "Attempt to setup unsupported torch mode " << avTorchMode; + }; + + auto setAvFlashModeSafe = [&captureDevice](AVCaptureFlashMode avFlashMode) { + if ([captureDevice isFlashModeSupported:avFlashMode]) + captureDevice.flashMode = avFlashMode; + else + qDebugCamera() << Q_FUNC_INFO << "Attempt to setup unsupported flash mode " << avFlashMode; + }; + if (!isFlashModeSupported(m_flashMode)) { qDebugCamera() << Q_FUNC_INFO << "unsupported mode" << m_flashMode; return false; @@ -192,7 +206,7 @@ bool AVFCameraFlashControl::applyFlashSettings() return false; } #endif - captureDevice.torchMode = AVCaptureTorchModeOff; + setAvTorchModeSafe(AVCaptureTorchModeOff); } #ifdef Q_OS_IOS if (![captureDevice isFlashAvailable]) { @@ -209,7 +223,7 @@ bool AVFCameraFlashControl::applyFlashSettings() return false; } #endif - captureDevice.flashMode = AVCaptureFlashModeOff; + setAvFlashModeSafe(AVCaptureFlashModeOff); } #ifdef Q_OS_IOS @@ -221,13 +235,13 @@ bool AVFCameraFlashControl::applyFlashSettings() } if (m_flashMode == QCameraExposure::FlashOff) - captureDevice.flashMode = AVCaptureFlashModeOff; + setAvFlashModeSafe(AVCaptureFlashModeOff); else if (m_flashMode == QCameraExposure::FlashOn) - captureDevice.flashMode = AVCaptureFlashModeOn; + setAvFlashModeSafe(AVCaptureFlashModeOn); else if (m_flashMode == QCameraExposure::FlashAuto) - captureDevice.flashMode = AVCaptureFlashModeAuto; + setAvFlashModeSafe(AVCaptureFlashModeAuto); else if (m_flashMode == QCameraExposure::FlashVideoLight) - captureDevice.torchMode = AVCaptureTorchModeOn; + setAvTorchModeSafe(AVCaptureTorchModeOn); return true; } diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index d5554c51c..a601bb5ac 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -110,6 +110,12 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext self->m_session = session; self->m_bufferIsLikelyToKeepUp = FALSE; + + m_playerLayer = [AVPlayerLayer playerLayerWithPlayer:nil]; + [m_playerLayer retain]; + m_playerLayer.videoGravity = AVLayerVideoGravityResizeAspectFill; + m_playerLayer.anchorPoint = CGPointMake(0.0f, 0.0f); + return self; } @@ -162,6 +168,10 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext [[NSNotificationCenter defaultCenter] removeObserver:self name:AVPlayerItemTimeJumpedNotification object:m_playerItem]; + for (AVPlayerItemOutput *output in m_playerItem.outputs) { + if ([output isKindOfClass:[AVPlayerItemVideoOutput class]]) + [m_playerItem removeOutput:output]; + } m_playerItem = 0; } if (m_player) { @@ -172,10 +182,6 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext [m_player release]; m_player = 0; } - if (m_playerLayer) { - [m_playerLayer release]; - m_playerLayer = 0; - } } - (void) prepareToPlayAsset:(AVURLAsset *)asset @@ -260,14 +266,8 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext [m_player setMuted:m_session->isMuted()]; } - //Create a new player layer if we don't have one already - if (!m_playerLayer) - { - m_playerLayer = [AVPlayerLayer playerLayerWithPlayer:m_player]; - [m_playerLayer retain]; - m_playerLayer.videoGravity = AVLayerVideoGravityResizeAspectFill; - m_playerLayer.anchorPoint = CGPointMake(0.0f, 0.0f); - } + //Assign the output layer to the new player + m_playerLayer.player = m_player; //Observe the AVPlayer "currentItem" property to find out when any //AVPlayer replaceCurrentItemWithPlayerItem: replacement will/did @@ -413,6 +413,7 @@ static void *AVFMediaPlayerSessionObserverCurrentItemDurationObservationContext } [m_mimeType release]; + [m_playerLayer release]; [super dealloc]; } diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index c5d7fe66c..ac67090a5 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -43,22 +43,48 @@ #include <QtCore/QObject> #include <QtGui/QImage> #include <QtGui/QOpenGLContext> +#include <QtGui/QOpenGLTextureBlitter> #include <QtCore/QSize> -#import "Metal/Metal.h" -#import "MetalKit/MetalKit.h" - -@class CARenderer; @class AVPlayerLayer; +@class AVPlayerItemVideoOutput; QT_BEGIN_NAMESPACE +class QOpenGLContext; class QOpenGLFramebufferObject; class QOpenGLShaderProgram; -class QWindow; -class QOpenGLContext; +class QOffscreenSurface; class QAbstractVideoSurface; +typedef struct __CVBuffer *CVBufferRef; +typedef CVBufferRef CVImageBufferRef; +typedef CVImageBufferRef CVPixelBufferRef; + +#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) + typedef struct __CVOpenGLESTextureCache *CVOpenGLESTextureCacheRef; + typedef CVImageBufferRef CVOpenGLESTextureRef; + // helpers to avoid boring if def + typedef CVOpenGLESTextureCacheRef CVOGLTextureCacheRef; + typedef CVOpenGLESTextureRef CVOGLTextureRef; + #define CVOGLTextureGetTarget CVOpenGLESTextureGetTarget + #define CVOGLTextureGetName CVOpenGLESTextureGetName + #define CVOGLTextureCacheCreate CVOpenGLESTextureCacheCreate + #define CVOGLTextureCacheCreateTextureFromImage CVOpenGLESTextureCacheCreateTextureFromImage + #define CVOGLTextureCacheFlush CVOpenGLESTextureCacheFlush +#else + typedef struct __CVOpenGLTextureCache *CVOpenGLTextureCacheRef; + typedef CVImageBufferRef CVOpenGLTextureRef; + // helpers to avoid boring if def + typedef CVOpenGLTextureCacheRef CVOGLTextureCacheRef; + typedef CVOpenGLTextureRef CVOGLTextureRef; + #define CVOGLTextureGetTarget CVOpenGLTextureGetTarget + #define CVOGLTextureGetName CVOpenGLTextureGetName + #define CVOGLTextureCacheCreate CVOpenGLTextureCacheCreate + #define CVOGLTextureCacheCreateTextureFromImage CVOpenGLTextureCacheCreateTextureFromImage + #define CVOGLTextureCacheFlush CVOpenGLTextureCacheFlush +#endif + class AVFVideoFrameRenderer : public QObject { public: @@ -66,57 +92,31 @@ public: virtual ~AVFVideoFrameRenderer(); - GLuint renderLayerToTexture(AVPlayerLayer *layer); - QImage renderLayerToImage(AVPlayerLayer *layer); - - static GLuint createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, - CVOpenGLTextureCacheRef cvglTextureCache, - CVPixelBufferRef cvPixelBufferRef, - CVOpenGLTextureRef cvOpenGLTextureRef); + void setPlayerLayer(AVPlayerLayer *layer); - static id<MTLTexture> createMetalTexture(id<MTLDevice> mtlDevice, - CVMetalTextureCacheRef cvMetalTextureCacheRef, - CVPixelBufferRef cvPixelBufferRef, - MTLPixelFormat pixelFormat, size_t width, size_t height, - CVMetalTextureRef cvMetalTextureRef); + CVOGLTextureRef renderLayerToTexture(AVPlayerLayer *layer, QSize *size); +#ifdef Q_OS_MACOS + GLuint renderLayerToFBO(AVPlayerLayer *layer, QSize *size); +#endif + QImage renderLayerToImage(AVPlayerLayer *layer, QSize *size); private: - QOpenGLFramebufferObject* initRenderer(AVPlayerLayer *layer); - void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); - void renderLayerToFBOCoreOpenGL(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); + void initRenderer(); + CVPixelBufferRef copyPixelBufferFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); + CVOGLTextureRef createCacheTextureFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); - CARenderer *m_videoLayerRenderer; - QAbstractVideoSurface *m_surface; - QOpenGLFramebufferObject *m_fbo[2]; - QOpenGLShaderProgram *m_shader = nullptr; - QWindow *m_offscreenSurface; QOpenGLContext *m_glContext; - QSize m_targetSize; - - bool m_useCoreProfile = false; - - // Shared pixel buffer - CVPixelBufferRef m_CVPixelBuffer; - - // OpenGL Texture - CVOpenGLTextureCacheRef m_CVGLTextureCache; - CVOpenGLTextureRef m_CVGLTexture; - CGLPixelFormatObj m_CGLPixelFormat; - GLuint m_textureName = 0; - - // Metal Texture - CVMetalTextureRef m_CVMTLTexture; - CVMetalTextureCacheRef m_CVMTLTextureCache; - id<MTLDevice> m_metalDevice = nil; - id<MTLTexture> m_metalTexture = nil; - - NSOpenGLContext *m_NSGLContext = nullptr; - - GLuint m_quadVao = 0; - GLuint m_quadVbos[2]; - - uint m_currentBuffer; + QOffscreenSurface *m_offscreenSurface; + QAbstractVideoSurface *m_surface; + CVOGLTextureCacheRef m_textureCache; + AVPlayerItemVideoOutput* m_videoOutput; bool m_isContextShared; + +#ifdef Q_OS_MACOS + QOpenGLFramebufferObject *m_fbo[2]; + uint m_currentFBO; + QOpenGLTextureBlitter m_blitter; +#endif }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 23e5497a2..a22ee2b82 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -41,14 +41,20 @@ #include <QtMultimedia/qabstractvideosurface.h> #include <QtGui/QOpenGLFramebufferObject> -#include <QtGui/QWindow> -#include <QOpenGLShaderProgram> -#include <QtPlatformHeaders/QCocoaNativeContext> +#include <QtGui/QOpenGLShaderProgram> +#include <QtGui/QOffscreenSurface> + +#include <QtCore/private/qcore_mac_p.h> #ifdef QT_DEBUG_AVF #include <QtCore/qdebug.h> #endif +#ifdef Q_OS_MACOS +#import <AppKit/AppKit.h> +#include <CoreVideo/CVOpenGLTextureCache.h> +#endif + #import <CoreVideo/CVBase.h> #import <AVFoundation/AVFoundation.h> @@ -56,15 +62,23 @@ QT_USE_NAMESPACE AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent) : QObject(parent) - , m_videoLayerRenderer(nullptr) - , m_surface(surface) - , m_offscreenSurface(nullptr) , m_glContext(nullptr) - , m_currentBuffer(1) + , m_offscreenSurface(nullptr) + , m_surface(surface) + , m_textureCache(nullptr) + , m_videoOutput(nullptr) , m_isContextShared(true) { + m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:@{ + (NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (NSString *)kCVPixelBufferOpenGLCompatibilityKey: @YES + }]; + [m_videoOutput setDelegate:nil queue:nil]; + +#ifdef Q_OS_MACOS m_fbo[0] = nullptr; m_fbo[1] = nullptr; +#endif } AVFVideoFrameRenderer::~AVFVideoFrameRenderer() @@ -73,355 +87,244 @@ AVFVideoFrameRenderer::~AVFVideoFrameRenderer() qDebug() << Q_FUNC_INFO; #endif - [m_videoLayerRenderer release]; - delete m_fbo[0]; - delete m_fbo[1]; + [m_videoOutput release]; + if (m_textureCache) + CFRelease(m_textureCache); delete m_offscreenSurface; delete m_glContext; - if (m_useCoreProfile) { - glDeleteVertexArrays(1, &m_quadVao); - glDeleteBuffers(2, m_quadVbos); - delete m_shader; - } +#ifdef Q_OS_MACOS + delete m_fbo[0]; + delete m_fbo[1]; +#endif } -GLuint AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) +#ifdef Q_OS_MACOS +GLuint AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QSize *size) { - //Is layer valid - if (!layer) + QCFType<CVOGLTextureRef> texture = renderLayerToTexture(layer, size); + if (!texture) return 0; - //If the glContext isn't shared, it doesn't make sense to return a texture for us - if (m_offscreenSurface && !m_isContextShared) - return 0; + Q_ASSERT(size); - QOpenGLFramebufferObject *fbo = initRenderer(layer); + // Do we have FBO's already? + if ((!m_fbo[0] && !m_fbo[0]) || (m_fbo[0]->size() != *size)) { + delete m_fbo[0]; + delete m_fbo[1]; + m_fbo[0] = new QOpenGLFramebufferObject(*size); + m_fbo[1] = new QOpenGLFramebufferObject(*size); + } - if (!fbo) - return 0; + // Switch buffer target + m_currentFBO = !m_currentFBO; + QOpenGLFramebufferObject *fbo = m_fbo[m_currentFBO]; - renderLayerToFBO(layer, fbo); - if (m_glContext) - m_glContext->doneCurrent(); + if (!fbo || !fbo->bind()) + return 0; - return fbo->texture(); -} + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); -QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) -{ - //Is layer valid - if (!layer) { - return QImage(); - } + glViewport(0, 0, size->width(), size->height()); - QOpenGLFramebufferObject *fbo = initRenderer(layer); + if (!m_blitter.isCreated()) + m_blitter.create(); - if (!fbo) - return QImage(); + m_blitter.bind(GL_TEXTURE_RECTANGLE); + m_blitter.blit(CVOpenGLTextureGetName(texture), QMatrix4x4(), QMatrix3x3()); + m_blitter.release(); - renderLayerToFBO(layer, fbo); - QImage fboImage = fbo->toImage(); - if (m_glContext) - m_glContext->doneCurrent(); + glFinish(); - return fboImage; + fbo->release(); + return fbo->texture(); } +#endif -QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *layer) +CVOGLTextureRef AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer, QSize *size) { + initRenderer(); - //Get size from AVPlayerLayer - m_targetSize = QSize(layer.bounds.size.width, layer.bounds.size.height); - - QOpenGLContext *shareContext = !m_glContext && m_surface - ? qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>()) - : nullptr; + // If the glContext isn't shared, it doesn't make sense to return a texture for us + if (!m_isContextShared) + return nullptr; - //Make sure we have an OpenGL context to make current - if ((shareContext && shareContext != QOpenGLContext::currentContext()) - || (!QOpenGLContext::currentContext() && !m_glContext)) { - - //Create Hidden QWindow surface to create context in this thread - delete m_offscreenSurface; - m_offscreenSurface = new QWindow(); - m_offscreenSurface->setSurfaceType(QWindow::OpenGLSurface); - //Needs geometry to be a valid surface, but size is not important - m_offscreenSurface->setGeometry(0, 0, 1, 1); - m_offscreenSurface->create(); - - delete m_glContext; - m_glContext = new QOpenGLContext(); - m_glContext->setFormat(m_offscreenSurface->requestedFormat()); + size_t width = 0, height = 0; + auto texture = createCacheTextureFromLayer(layer, width, height); + if (size) + *size = QSize(width, height); + return texture; +} - if (shareContext) { - m_glContext->setShareContext(shareContext); - m_isContextShared = true; - } else { +CVPixelBufferRef AVFVideoFrameRenderer::copyPixelBufferFromLayer(AVPlayerLayer *layer, + size_t& width, size_t& height) +{ + //Is layer valid + if (!layer) { #ifdef QT_DEBUG_AVF - qWarning("failed to get Render Thread context"); + qWarning("copyPixelBufferFromLayer: invalid layer"); #endif - m_isContextShared = false; - } - if (!m_glContext->create()) { - qWarning("failed to create QOpenGLContext"); - return nullptr; - } - - // CARenderer must be re-created with different current context, so release it now. - // See lines below where m_videoLayerRenderer is constructed. - if (m_videoLayerRenderer) { - [m_videoLayerRenderer release]; - m_videoLayerRenderer = nullptr; - } - - if (m_useCoreProfile) { - glDeleteVertexArrays(1, &m_quadVao); - glDeleteBuffers(2, m_quadVbos); - delete m_shader; - m_shader = nullptr; - } + return nullptr; } - //Need current context - if (m_glContext) - m_glContext->makeCurrent(m_offscreenSurface); + AVPlayerItem *item = layer.player.currentItem; + if (![item.outputs containsObject:m_videoOutput]) + [item addOutput:m_videoOutput]; - if (!m_metalDevice) - m_metalDevice = MTLCreateSystemDefaultDevice(); + CFTimeInterval currentCAFrameTime = CACurrentMediaTime(); + CMTime currentCMFrameTime = [m_videoOutput itemTimeForHostTime:currentCAFrameTime]; - if (@available(macOS 10.13, *)) { - m_useCoreProfile = m_metalDevice && (QOpenGLContext::currentContext()->format().profile() == - QSurfaceFormat::CoreProfile); - } else { - m_useCoreProfile = false; - } - - // Create the CARenderer if needed for no Core OpenGL - if (!m_videoLayerRenderer) { - if (!m_useCoreProfile) { - m_videoLayerRenderer = [CARenderer rendererWithCGLContext: CGLGetCurrentContext() - options: nil]; - [m_videoLayerRenderer retain]; - } else if (@available(macOS 10.13, *)) { - // This is always true when m_useCoreProfile is true, but the compiler wants the check - // anyway - // Setup Core OpenGL shader, VAO, VBOs and metal renderer - m_shader = new QOpenGLShaderProgram(); - m_shader->create(); - if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Vertex, R"(#version 150 core - in vec2 qt_VertexPosition; - in vec2 qt_VertexTexCoord; - out vec2 qt_TexCoord; - void main() - { - qt_TexCoord = qt_VertexTexCoord; - gl_Position = vec4(qt_VertexPosition, 0.0f, 1.0f); - })")) { - qCritical() << "Vertex shader compilation failed" << m_shader->log(); - } - if (!m_shader->addShaderFromSourceCode(QOpenGLShader::Fragment, R"(#version 150 core - in vec2 qt_TexCoord; - out vec4 fragColor; - uniform sampler2DRect videoFrame; - void main(void) - { - ivec2 textureDim = textureSize(videoFrame); - fragColor = texture(videoFrame, qt_TexCoord * textureDim); - })")) { - qCritical() << "Fragment shader compilation failed" << m_shader->log(); - } - - // Setup quad where the video frame will be attached - GLfloat vertices[] = { - -1.0f, -1.0f, - 1.0f, -1.0f, - -1.0f, 1.0f, - 1.0f, 1.0f, - }; - - GLfloat uvs[] = { - 0.0f, 0.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, - }; - - glGenVertexArrays(1, &m_quadVao); - glBindVertexArray(m_quadVao); - - // Create vertex buffer objects for vertices - glGenBuffers(2, m_quadVbos); - - // Setup vertices - glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[0]); - glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); - glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); - glEnableVertexAttribArray(0); - - // Setup uvs - glBindBuffer(GL_ARRAY_BUFFER, m_quadVbos[1]); - glBufferData(GL_ARRAY_BUFFER, sizeof(uvs), uvs, GL_STATIC_DRAW); - glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(float), nullptr); - glEnableVertexAttribArray(1); - - glBindVertexArray(0); - - // Setup shared Metal/OpenGL pixel buffer and textures - m_NSGLContext = static_cast<QCocoaNativeContext*>((QOpenGLContext::currentContext()->nativeHandle().data()))->context(); - m_CGLPixelFormat = m_NSGLContext.pixelFormat.CGLPixelFormatObj; - - NSDictionary* cvBufferProperties = @{ - static_cast<NSString*>(kCVPixelBufferOpenGLCompatibilityKey) : @YES, - static_cast<NSString*>(kCVPixelBufferMetalCompatibilityKey): @YES, - }; - - CVPixelBufferCreate(kCFAllocatorDefault, static_cast<size_t>(m_targetSize.width()), - static_cast<size_t>(m_targetSize.height()), kCVPixelFormatType_32BGRA, - static_cast<CFDictionaryRef>(cvBufferProperties), &m_CVPixelBuffer); - - m_textureName = createGLTexture(reinterpret_cast<CGLContextObj>(m_NSGLContext.CGLContextObj), - m_CGLPixelFormat, m_CVGLTextureCache, m_CVPixelBuffer, - m_CVGLTexture); - m_metalTexture = createMetalTexture(m_metalDevice, m_CVMTLTextureCache, m_CVPixelBuffer, - MTLPixelFormatBGRA8Unorm, - static_cast<size_t>(m_targetSize.width()), - static_cast<size_t>(m_targetSize.height()), - m_CVMTLTexture); - - m_videoLayerRenderer = [CARenderer rendererWithMTLTexture:m_metalTexture options:nil]; - [m_videoLayerRenderer retain]; - } - } + // Happens when buffering / loading + if (CMTimeCompare(currentCMFrameTime, kCMTimeZero) < 0) + return nullptr; - //Set/Change render source if needed - if (m_videoLayerRenderer.layer != layer) { - m_videoLayerRenderer.layer = layer; - m_videoLayerRenderer.bounds = layer.bounds; - } + if (![m_videoOutput hasNewPixelBufferForItemTime:currentCMFrameTime]) + return nullptr; - //Do we have FBO's already? - if ((!m_fbo[0] && !m_fbo[0]) || (m_fbo[0]->size() != m_targetSize)) { - delete m_fbo[0]; - delete m_fbo[1]; - m_fbo[0] = new QOpenGLFramebufferObject(m_targetSize); - m_fbo[1] = new QOpenGLFramebufferObject(m_targetSize); + CVPixelBufferRef pixelBuffer = [m_videoOutput copyPixelBufferForItemTime:currentCMFrameTime + itemTimeForDisplay:nil]; + if (!pixelBuffer) { +#ifdef QT_DEBUG_AVF + qWarning("copyPixelBufferForItemTime returned nil"); + CMTimeShow(currentCMFrameTime); +#endif + return nullptr; } - //Switch buffer target - m_currentBuffer = !m_currentBuffer; - return m_fbo[m_currentBuffer]; + width = CVPixelBufferGetWidth(pixelBuffer); + height = CVPixelBufferGetHeight(pixelBuffer); + return pixelBuffer; } -void AVFVideoFrameRenderer::renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo) +CVOGLTextureRef AVFVideoFrameRenderer::createCacheTextureFromLayer(AVPlayerLayer *layer, + size_t& width, size_t& height) { - //Start Rendering - //NOTE: This rendering method will NOT work on iOS as there is no CARenderer in iOS - if (!fbo->bind()) { - qWarning("AVFVideoRender FBO failed to bind"); - return; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + + if (!pixelBuffer) + return nullptr; + + CVOGLTextureCacheFlush(m_textureCache, 0); + + CVOGLTextureRef texture = nullptr; +#ifdef Q_OS_MACOS + CVReturn err = CVOpenGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + m_textureCache, + pixelBuffer, + nil, + &texture); +#else + CVReturn err = CVOGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_textureCache, pixelBuffer, nullptr, + GL_TEXTURE_2D, GL_RGBA, + (GLsizei) width, (GLsizei) height, + GL_BGRA, GL_UNSIGNED_BYTE, 0, + &texture); +#endif + + if (!texture || err) { + qWarning() << "CVOGLTextureCacheCreateTextureFromImage failed error:" << err << m_textureCache; } - glClearColor(0.0f, 0.0f, 0.0f, 1.0f); - glClear(GL_COLOR_BUFFER_BIT); + CVPixelBufferRelease(pixelBuffer); - glViewport(0, 0, m_targetSize.width(), m_targetSize.height()); + return texture; +} - if (m_useCoreProfile) { - CGLLockContext(m_NSGLContext.CGLContextObj); - m_shader->bind(); - glBindVertexArray(m_quadVao); - } else { - glMatrixMode(GL_PROJECTION); - glPushMatrix(); - glLoadIdentity(); +QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer, QSize *size) +{ + size_t width = 0; + size_t height = 0; + CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); + if (size) + *size = QSize(width, height); - // Render to FBO with inverted Y - glOrtho(0.0, m_targetSize.width(), 0.0, m_targetSize.height(), 0.0, 1.0); + if (!pixelBuffer) + return QImage(); - glMatrixMode(GL_MODELVIEW); - glPushMatrix(); - glLoadIdentity(); + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixelFormat != kCVPixelFormatType_32BGRA) { +#ifdef QT_DEBUG_AVF + qWarning("CVPixelBuffer format is not BGRA32 (got: %d)", static_cast<quint32>(pixelFormat)); +#endif + return QImage(); } - [m_videoLayerRenderer beginFrameAtTime:CACurrentMediaTime() timeStamp:nullptr]; - [m_videoLayerRenderer addUpdateRect:layer.bounds]; - [m_videoLayerRenderer render]; - [m_videoLayerRenderer endFrame]; - - if (m_useCoreProfile) { - glActiveTexture(0); - glBindTexture(GL_TEXTURE_RECTANGLE, m_textureName); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + char *data = (char *)CVPixelBufferGetBaseAddress(pixelBuffer); + size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer); - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + // format here is not relevant, only using for storage + QImage img = QImage(width, height, QImage::Format_ARGB32); + for (size_t j = 0; j < height; j++) { + memcpy(img.scanLine(j), data, width * 4); + data += stride; + } - glBindTexture(GL_TEXTURE_RECTANGLE, 0); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + CVPixelBufferRelease(pixelBuffer); + return img; +} - glBindVertexArray(0); +void AVFVideoFrameRenderer::initRenderer() +{ + // even for using a texture directly, we need to be able to make a context current, + // so we need an offscreen, and we shouldn't assume we can make the surface context + // current on that offscreen, so use our own (sharing with it). Slightly + // excessive but no performance penalty and makes the QImage path easier to maintain - m_shader->release(); + //Make sure we have an OpenGL context to make current + if (!m_glContext) { + //Create OpenGL context and set share context from surface + QOpenGLContext *shareContext = nullptr; + if (m_surface) + shareContext = qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>()); - CGLFlushDrawable(m_NSGLContext.CGLContextObj); - CGLUnlockContext(m_NSGLContext.CGLContextObj); - } else { - glMatrixMode(GL_MODELVIEW); - glPopMatrix(); - glMatrixMode(GL_PROJECTION); - glPopMatrix(); + m_glContext = new QOpenGLContext(); + if (shareContext) { + m_glContext->setShareContext(shareContext); + m_isContextShared = true; + } else { +#ifdef QT_DEBUG_AVF + qWarning("failed to get Render Thread context"); +#endif + m_isContextShared = false; + } + if (!m_glContext->create()) { +#ifdef QT_DEBUG_AVF + qWarning("failed to create QOpenGLContext"); +#endif + return; + } } - glFinish(); //Rendering needs to be done before passing texture to video frame - - fbo->release(); -} + if (!m_offscreenSurface) { + m_offscreenSurface = new QOffscreenSurface(); + m_offscreenSurface->setFormat(m_glContext->format()); + m_offscreenSurface->create(); + } -GLuint AVFVideoFrameRenderer::createGLTexture(CGLContextObj cglContextObj, CGLPixelFormatObj cglPixelFormtObj, CVOpenGLTextureCacheRef cvglTextureCache, - CVPixelBufferRef cvPixelBufferRef, CVOpenGLTextureRef cvOpenGLTextureRef) -{ - CVReturn cvret; - // Create an OpenGL CoreVideo texture cache from the pixel buffer. - cvret = CVOpenGLTextureCacheCreate( - kCFAllocatorDefault, - nil, - cglContextObj, - cglPixelFormtObj, - nil, - &cvglTextureCache); - - // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache. - cvret = CVOpenGLTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - cvglTextureCache, - cvPixelBufferRef, - nil, - &cvOpenGLTextureRef); - - // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image. - return CVOpenGLTextureGetName(cvOpenGLTextureRef); -} + // Need current context + m_glContext->makeCurrent(m_offscreenSurface); + + if (!m_textureCache) { +#ifdef Q_OS_MACOS + auto *currentContext = NSOpenGLContext.currentContext; + // Create an OpenGL CoreVideo texture cache from the pixel buffer. + auto err = CVOpenGLTextureCacheCreate( + kCFAllocatorDefault, + nullptr, + currentContext.CGLContextObj, + currentContext.pixelFormat.CGLPixelFormatObj, + nil, + &m_textureCache); +#else + CVReturn err = CVOGLTextureCacheCreate(kCFAllocatorDefault, nullptr, + [EAGLContext currentContext], + nullptr, &m_textureCache); +#endif + if (err) + qWarning("Error at CVOGLTextureCacheCreate %d", err); + } -id<MTLTexture> AVFVideoFrameRenderer::createMetalTexture(id<MTLDevice> mtlDevice, CVMetalTextureCacheRef cvMetalTextureCacheRef, CVPixelBufferRef cvPixelBufferRef, - MTLPixelFormat pixelFormat, size_t width, size_t height, CVMetalTextureRef cvMetalTextureRef) -{ - CVReturn cvret; - // Create a Metal Core Video texture cache from the pixel buffer. - cvret = CVMetalTextureCacheCreate( - kCFAllocatorDefault, - nil, - mtlDevice, - nil, - &cvMetalTextureCacheRef); - - // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. - cvret = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - cvMetalTextureCacheRef, - cvPixelBufferRef, nil, - pixelFormat, - width, height, - 0, - &cvMetalTextureRef); - - // Get a Metal texture using the CoreVideo Metal texture reference. - return CVMetalTextureGetTexture(cvMetalTextureRef); } diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h deleted file mode 100644 index d9f6baa7e..000000000 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.h +++ /dev/null @@ -1,113 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef AVFVIDEOFRAMERENDERER_H -#define AVFVIDEOFRAMERENDERER_H - -#include <QtCore/QObject> -#include <QtGui/QImage> -#include <QtGui/QOpenGLContext> -#include <QtCore/QSize> - -@class AVPlayerLayer; -@class AVPlayerItemVideoOutput; - -QT_BEGIN_NAMESPACE - -class QOpenGLContext; -class QOpenGLFramebufferObject; -class QOpenGLShaderProgram; -class QOffscreenSurface; -class QAbstractVideoSurface; - -typedef struct __CVBuffer *CVBufferRef; -typedef CVBufferRef CVImageBufferRef; -typedef CVImageBufferRef CVPixelBufferRef; -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -typedef struct __CVOpenGLESTextureCache *CVOpenGLESTextureCacheRef; -typedef CVImageBufferRef CVOpenGLESTextureRef; -// helpers to avoid boring if def -typedef CVOpenGLESTextureCacheRef CVOGLTextureCacheRef; -typedef CVOpenGLESTextureRef CVOGLTextureRef; -#define CVOGLTextureGetTarget CVOpenGLESTextureGetTarget -#define CVOGLTextureGetName CVOpenGLESTextureGetName -#define CVOGLTextureCacheCreate CVOpenGLESTextureCacheCreate -#define CVOGLTextureCacheCreateTextureFromImage CVOpenGLESTextureCacheCreateTextureFromImage -#define CVOGLTextureCacheFlush CVOpenGLESTextureCacheFlush -#else -typedef struct __CVOpenGLTextureCache *CVOpenGLTextureCacheRef; -typedef CVImageBufferRef CVOpenGLTextureRef; -// helpers to avoid boring if def -typedef CVOpenGLTextureCacheRef CVOGLTextureCacheRef; -typedef CVOpenGLTextureRef CVOGLTextureRef; -#define CVOGLTextureGetTarget CVOpenGLTextureGetTarget -#define CVOGLTextureGetName CVOpenGLTextureGetName -#define CVOGLTextureCacheCreate CVOpenGLTextureCacheCreate -#define CVOGLTextureCacheCreateTextureFromImage CVOpenGLTextureCacheCreateTextureFromImage -#define CVOGLTextureCacheFlush CVOpenGLTextureCacheFlush -#endif - -class AVFVideoFrameRenderer : public QObject -{ -public: - AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent = nullptr); - - virtual ~AVFVideoFrameRenderer(); - - void setPlayerLayer(AVPlayerLayer *layer); - - CVOGLTextureRef renderLayerToTexture(AVPlayerLayer *layer); - QImage renderLayerToImage(AVPlayerLayer *layer); - -private: - void initRenderer(); - CVPixelBufferRef copyPixelBufferFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); - CVOGLTextureRef createCacheTextureFromLayer(AVPlayerLayer *layer, size_t& width, size_t& height); - - QOpenGLContext *m_glContext; - QOffscreenSurface *m_offscreenSurface; - QAbstractVideoSurface *m_surface; - CVOGLTextureCacheRef m_textureCache; - AVPlayerItemVideoOutput* m_videoOutput; - bool m_isContextShared; -}; - -QT_END_NAMESPACE - -#endif // AVFVIDEOFRAMERENDERER_H diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm deleted file mode 100644 index 70e402e6c..000000000 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer_ios.mm +++ /dev/null @@ -1,261 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "avfvideoframerenderer_ios.h" - -#include <QtMultimedia/qabstractvideosurface.h> -#include <QtGui/QOpenGLFramebufferObject> -#include <QtGui/QOpenGLShaderProgram> -#include <QtGui/QOffscreenSurface> - -#ifdef QT_DEBUG_AVF -#include <QtCore/qdebug.h> -#endif - -#import <CoreVideo/CVBase.h> -#import <AVFoundation/AVFoundation.h> -QT_USE_NAMESPACE - -AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent) - : QObject(parent) - , m_glContext(nullptr) - , m_offscreenSurface(nullptr) - , m_surface(surface) - , m_textureCache(nullptr) - , m_videoOutput(nullptr) - , m_isContextShared(true) -{ -} - -AVFVideoFrameRenderer::~AVFVideoFrameRenderer() -{ -#ifdef QT_DEBUG_AVF - qDebug() << Q_FUNC_INFO; -#endif - - [m_videoOutput release]; // sending to nil is fine - if (m_textureCache) - CFRelease(m_textureCache); - delete m_offscreenSurface; - delete m_glContext; -} - -void AVFVideoFrameRenderer::setPlayerLayer(AVPlayerLayer *layer) -{ - Q_UNUSED(layer) - if (m_videoOutput) { - [m_videoOutput release]; - m_videoOutput = nullptr; - // will be re-created in first call to copyPixelBufferFromLayer - } -} - -CVOGLTextureRef AVFVideoFrameRenderer::renderLayerToTexture(AVPlayerLayer *layer) -{ - initRenderer(); - - // If the glContext isn't shared, it doesn't make sense to return a texture for us - if (!m_isContextShared) - return nullptr; - - size_t dummyWidth = 0, dummyHeight = 0; - return createCacheTextureFromLayer(layer, dummyWidth, dummyHeight); -} - -static NSString* const AVF_PIXEL_FORMAT_KEY = (NSString*)kCVPixelBufferPixelFormatTypeKey; -static NSNumber* const AVF_PIXEL_FORMAT_VALUE = [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]; -static NSDictionary* const AVF_OUTPUT_SETTINGS = [NSDictionary dictionaryWithObject:AVF_PIXEL_FORMAT_VALUE forKey:AVF_PIXEL_FORMAT_KEY]; - - -CVPixelBufferRef AVFVideoFrameRenderer::copyPixelBufferFromLayer(AVPlayerLayer *layer, - size_t& width, size_t& height) -{ - //Is layer valid - if (!layer) { -#ifdef QT_DEBUG_AVF - qWarning("copyPixelBufferFromLayer: invalid layer"); -#endif - return nullptr; - } - - if (!m_videoOutput) { - m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:AVF_OUTPUT_SETTINGS]; - [m_videoOutput setDelegate:nil queue:nil]; - AVPlayerItem * item = [[layer player] currentItem]; - [item addOutput:m_videoOutput]; - } - - CFTimeInterval currentCAFrameTime = CACurrentMediaTime(); - CMTime currentCMFrameTime = [m_videoOutput itemTimeForHostTime:currentCAFrameTime]; - // happens when buffering / loading - if (CMTimeCompare(currentCMFrameTime, kCMTimeZero) < 0) { - return nullptr; - } - - CVPixelBufferRef pixelBuffer = [m_videoOutput copyPixelBufferForItemTime:currentCMFrameTime - itemTimeForDisplay:nil]; - if (!pixelBuffer) { -#ifdef QT_DEBUG_AVF - qWarning("copyPixelBufferForItemTime returned nil"); - CMTimeShow(currentCMFrameTime); -#endif - return nullptr; - } - - width = CVPixelBufferGetWidth(pixelBuffer); - height = CVPixelBufferGetHeight(pixelBuffer); - return pixelBuffer; -} - -CVOGLTextureRef AVFVideoFrameRenderer::createCacheTextureFromLayer(AVPlayerLayer *layer, - size_t& width, size_t& height) -{ - CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); - - if (!pixelBuffer) - return nullptr; - - CVOGLTextureCacheFlush(m_textureCache, 0); - - CVOGLTextureRef texture = nullptr; - CVReturn err = CVOGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, m_textureCache, pixelBuffer, nullptr, - GL_TEXTURE_2D, GL_RGBA, - (GLsizei) width, (GLsizei) height, - GL_BGRA, GL_UNSIGNED_BYTE, 0, - &texture); - - if (!texture || err) { -#ifdef QT_DEBUG_AVF - qWarning("CVOGLTextureCacheCreateTextureFromImage failed (error: %d)", err); -#endif - } - - CVPixelBufferRelease(pixelBuffer); - - return texture; -} - -QImage AVFVideoFrameRenderer::renderLayerToImage(AVPlayerLayer *layer) -{ - size_t width = 0; - size_t height = 0; - CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(layer, width, height); - - if (!pixelBuffer) - return QImage(); - - OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); - if (pixelFormat != kCVPixelFormatType_32BGRA) { -#ifdef QT_DEBUG_AVF - qWarning("CVPixelBuffer format is not BGRA32 (got: %d)", static_cast<quint32>(pixelFormat)); -#endif - return QImage(); - } - - CVPixelBufferLockBaseAddress(pixelBuffer, 0); - char *data = (char *)CVPixelBufferGetBaseAddress(pixelBuffer); - size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer); - - // format here is not relevant, only using for storage - QImage img = QImage(width, height, QImage::Format_ARGB32); - for (size_t j = 0; j < height; j++) { - memcpy(img.scanLine(j), data, width * 4); - data += stride; - } - - CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); - CVPixelBufferRelease(pixelBuffer); - return img; -} - -void AVFVideoFrameRenderer::initRenderer() -{ - // even for using a texture directly, we need to be able to make a context current, - // so we need an offscreen, and we shouldn't assume we can make the surface context - // current on that offscreen, so use our own (sharing with it). Slightly - // excessive but no performance penalty and makes the QImage path easier to maintain - - //Make sure we have an OpenGL context to make current - if (!m_glContext) { - //Create OpenGL context and set share context from surface - QOpenGLContext *shareContext = nullptr; - if (m_surface) { - shareContext = qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>()); - } - - m_glContext = new QOpenGLContext(); - if (shareContext) { - m_glContext->setShareContext(shareContext); - m_isContextShared = true; - } else { -#ifdef QT_DEBUG_AVF - qWarning("failed to get Render Thread context"); -#endif - m_isContextShared = false; - } - if (!m_glContext->create()) { -#ifdef QT_DEBUG_AVF - qWarning("failed to create QOpenGLContext"); -#endif - return; - } - } - - if (!m_offscreenSurface) { - m_offscreenSurface = new QOffscreenSurface(); - m_offscreenSurface->setFormat(m_glContext->format()); - m_offscreenSurface->create(); - } - - //Need current context - m_glContext->makeCurrent(m_offscreenSurface); - - if (!m_textureCache) { - // Create a new open gl texture cache - CVReturn err = CVOGLTextureCacheCreate(kCFAllocatorDefault, nullptr, - [EAGLContext currentContext], - nullptr, &m_textureCache); - if (err) { - #ifdef QT_DEBUG_AVF - qWarning("Error at CVOGLTextureCacheCreate %d", err); - #endif - } - } - -} diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h index 85dc19d31..a88573eaa 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.h @@ -41,6 +41,8 @@ #define AVFVIDEORENDERERCONTROL_H #include <QtMultimedia/QVideoRendererControl> +#include <QtMultimedia/qabstractvideobuffer.h> + #include <QtCore/QMutex> #include <QtCore/QSize> @@ -82,8 +84,7 @@ private: AVFVideoFrameRenderer *m_frameRenderer; AVFDisplayLink *m_displayLink; - QSize m_nativeSize; - bool m_enableOpenGL; + QAbstractVideoBuffer::HandleType m_surfaceType = QAbstractVideoBuffer::NoHandle; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm index 63bdee4f5..f299d5f86 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideorenderercontrol.mm @@ -40,11 +40,7 @@ #include "avfvideorenderercontrol.h" #include "avfdisplaylink.h" -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -#include "avfvideoframerenderer_ios.h" -#else #include "avfvideoframerenderer.h" -#endif #include <QtMultimedia/qabstractvideobuffer.h> #include <QtMultimedia/qabstractvideosurface.h> @@ -58,69 +54,52 @@ QT_USE_NAMESPACE -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) -class TextureCacheVideoBuffer : public QAbstractVideoBuffer +class TextureVideoBuffer : public QAbstractVideoBuffer { public: - TextureCacheVideoBuffer(CVOGLTextureRef texture) - : QAbstractVideoBuffer(GLTextureHandle) + TextureVideoBuffer(GLuint texture, QAbstractVideoBuffer::HandleType type) + : QAbstractVideoBuffer(type) , m_texture(texture) {} - virtual ~TextureCacheVideoBuffer() - { - // absolutely critical that we drop this - // reference of textures will stay in the cache - CFRelease(m_texture); - } - MapMode mapMode() const { return NotMapped; } uchar *map(MapMode, int*, int*) { return nullptr; } void unmap() {} QVariant handle() const { - GLuint texId = CVOGLTextureGetName(m_texture); - return QVariant::fromValue<unsigned int>(texId); + return QVariant::fromValue<unsigned int>(m_texture); } private: - CVOGLTextureRef m_texture; + GLuint m_texture; }; -#else -class TextureVideoBuffer : public QAbstractVideoBuffer + +class CoreVideoTextureVideoBuffer : public TextureVideoBuffer { public: - TextureVideoBuffer(GLuint tex) - : QAbstractVideoBuffer(GLTextureHandle) - , m_texture(tex) + CoreVideoTextureVideoBuffer(CVOGLTextureRef texture, QAbstractVideoBuffer::HandleType type) + : TextureVideoBuffer(CVOGLTextureGetName(texture), type) + , m_coreVideoTexture(texture) {} - virtual ~TextureVideoBuffer() + virtual ~CoreVideoTextureVideoBuffer() { - } - - MapMode mapMode() const { return NotMapped; } - uchar *map(MapMode, int*, int*) { return 0; } - void unmap() {} - - QVariant handle() const - { - return QVariant::fromValue<unsigned int>(m_texture); + // absolutely critical that we drop this + // reference of textures will stay in the cache + CFRelease(m_coreVideoTexture); } private: - GLuint m_texture; + CVOGLTextureRef m_coreVideoTexture; }; -#endif + AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent) : QVideoRendererControl(parent) , m_surface(nullptr) , m_playerLayer(nullptr) , m_frameRenderer(nullptr) - , m_enableOpenGL(false) - { m_displayLink = new AVFDisplayLink(this); connect(m_displayLink, SIGNAL(tick(CVTimeStamp)), SLOT(updateVideoFrame(CVTimeStamp))); @@ -170,18 +149,26 @@ void AVFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) //Surface changed, so we need a new frame renderer m_frameRenderer = new AVFVideoFrameRenderer(m_surface, this); -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - if (m_playerLayer) { - m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(m_playerLayer)); - } -#endif - //Check for needed formats to render as OpenGL Texture - auto handleGlEnabled = [this] { - m_enableOpenGL = m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGR32); + auto updateSurfaceType = [this] { + auto preferredOpenGLSurfaceTypes = { +#ifdef Q_OS_MACOS + QAbstractVideoBuffer::GLTextureRectangleHandle, // GL_TEXTURE_RECTANGLE +#endif + QAbstractVideoBuffer::GLTextureHandle // GL_TEXTURE_2D + }; + + for (auto surfaceType : preferredOpenGLSurfaceTypes) { + auto supportedFormats = m_surface->supportedPixelFormats(surfaceType); + if (supportedFormats.contains(QVideoFrame::Format_BGR32)) { + m_surfaceType = surfaceType; + return; + } + m_surfaceType = QAbstractVideoBuffer::NoHandle; // QImage + } }; - handleGlEnabled(); - connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, handleGlEnabled); + updateSurfaceType(); + connect(m_surface, &QAbstractVideoSurface::supportedFormatsChanged, this, updateSurfaceType); //If we already have a layer, but changed surfaces start rendering again if (m_playerLayer && !m_displayLink->isActive()) { @@ -204,12 +191,6 @@ void AVFVideoRendererControl::setLayer(void *playerLayer) if (m_surface && m_surface->isActive()) m_surface->stop(); -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - if (m_frameRenderer) { - m_frameRenderer->setPlayerLayer(static_cast<AVPlayerLayer*>(playerLayer)); - } -#endif - //If there is no layer to render, stop scheduling updates if (m_playerLayer == nullptr) { m_displayLink->stop(); @@ -238,36 +219,39 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) if (!playerLayer.readyForDisplay) return; - if (m_enableOpenGL) { -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) - CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer); - - //Make sure we got a valid texture - if (tex == nullptr) - return; - - QAbstractVideoBuffer *buffer = new TextureCacheVideoBuffer(tex); + if (m_surfaceType == QAbstractVideoBuffer::GLTextureHandle + || m_surfaceType == QAbstractVideoBuffer::GLTextureRectangleHandle) { + QSize size; + QAbstractVideoBuffer *buffer = nullptr; + +#ifdef Q_OS_MACOS + if (m_surfaceType == QAbstractVideoBuffer::GLTextureRectangleHandle) { + // Render to GL_TEXTURE_RECTANGLE directly + if (CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer, &size)) + buffer = new CoreVideoTextureVideoBuffer(tex, m_surfaceType); + } else { + // Render to GL_TEXTURE_2D via FBO + if (GLuint tex = m_frameRenderer->renderLayerToFBO(playerLayer, &size)) + buffer = new TextureVideoBuffer(tex, m_surfaceType); + } #else - GLuint tex = m_frameRenderer->renderLayerToTexture(playerLayer); - //Make sure we got a valid texture - if (tex == 0) + Q_ASSERT(m_surfaceType != QAbstractVideoBuffer::GLTextureRectangleHandle); + // Render to GL_TEXTURE_2D directly + if (CVOGLTextureRef tex = m_frameRenderer->renderLayerToTexture(playerLayer, &size)) + buffer = new CoreVideoTextureVideoBuffer(tex, m_surfaceType); +#endif + if (!buffer) return; - QAbstractVideoBuffer *buffer = new TextureVideoBuffer(tex); -#endif - QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); + QVideoFrame frame = QVideoFrame(buffer, size, QVideoFrame::Format_BGR32); if (m_surface && frame.isValid()) { if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) m_surface->stop(); if (!m_surface->isActive()) { - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::GLTextureHandle); -#if defined(Q_OS_IOS) || defined(Q_OS_TVOS) + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), m_surfaceType); format.setScanLineDirection(QVideoSurfaceFormat::TopToBottom); -#else - format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); -#endif if (!m_surface->start(format)) { //Surface doesn't support GLTextureHandle qWarning("Failed to activate video surface"); @@ -279,20 +263,21 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) } } else { //fallback to rendering frames to QImages - QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer); + QSize size; + QImage frameData = m_frameRenderer->renderLayerToImage(playerLayer, &size); if (frameData.isNull()) { return; } QAbstractVideoBuffer *buffer = new QImageVideoBuffer(frameData); - QVideoFrame frame = QVideoFrame(buffer, m_nativeSize, QVideoFrame::Format_ARGB32); + QVideoFrame frame = QVideoFrame(buffer, size, QVideoFrame::Format_ARGB32); if (m_surface && frame.isValid()) { if (m_surface->isActive() && m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()) m_surface->stop(); if (!m_surface->isActive()) { - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), QAbstractVideoBuffer::NoHandle); + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), m_surfaceType); if (!m_surface->start(format)) { qWarning("Failed to activate video surface"); @@ -308,7 +293,4 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) void AVFVideoRendererControl::setupVideoOutput() { - AVPlayerLayer *playerLayer = static_cast<AVPlayerLayer*>(m_playerLayer); - if (playerLayer) - m_nativeSize = QSize(playerLayer.bounds.size.width, playerLayer.bounds.size.height); } diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index 174220f37..f71e0c3b3 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -39,34 +39,23 @@ OBJECTIVE_SOURCES += \ avfvideowidget.mm } -ios|tvos { - qtConfig(opengl) { - HEADERS += \ - avfvideoframerenderer_ios.h \ - avfvideorenderercontrol.h \ - avfdisplaylink.h +qtConfig(opengl) { + HEADERS += \ + avfvideoframerenderer.h \ + avfvideorenderercontrol.h \ + avfdisplaylink.h - OBJECTIVE_SOURCES += \ - avfvideoframerenderer_ios.mm \ - avfvideorenderercontrol.mm \ - avfdisplaylink.mm - } + OBJECTIVE_SOURCES += \ + avfvideoframerenderer.mm \ + avfvideorenderercontrol.mm \ + avfdisplaylink.mm +} + +ios|tvos { LIBS += -framework Foundation } else { INCLUDEPATH += $$[QT_INSTALL_HEADERS] LIBS += -framework AppKit -framework Metal - - qtConfig(opengl) { - HEADERS += \ - avfvideoframerenderer.h \ - avfvideorenderercontrol.h \ - avfdisplaylink.h - - OBJECTIVE_SOURCES += \ - avfvideoframerenderer.mm \ - avfvideorenderercontrol.mm \ - avfdisplaylink.mm - } } OTHER_FILES += \ diff --git a/src/plugins/common/evr/evrcustompresenter.cpp b/src/plugins/common/evr/evrcustompresenter.cpp index b2dd0426c..3405f6373 100644 --- a/src/plugins/common/evr/evrcustompresenter.cpp +++ b/src/plugins/common/evr/evrcustompresenter.cpp @@ -1065,6 +1065,13 @@ void EVRCustomPresenter::setSurface(QAbstractVideoSurface *surface) supportedFormatsChanged(); } +void EVRCustomPresenter::setCropRect(QRect cropRect) +{ + m_mutex.lock(); + m_cropRect = cropRect; + m_mutex.unlock(); +} + HRESULT EVRCustomPresenter::configureMixer(IMFTransform *mixer) { // Set the zoom rectangle (ie, the source clipping rectangle). @@ -1355,13 +1362,30 @@ HRESULT EVRCustomPresenter::createOptimalVideoType(IMFMediaType *proposedType, I hr = proposedType->GetUINT64(MF_MT_FRAME_SIZE, &size); width = int(HI32(size)); height = int(LO32(size)); - rcOutput.left = 0; - rcOutput.top = 0; - rcOutput.right = width; - rcOutput.bottom = height; + + if (m_cropRect.isValid()) { + rcOutput.left = m_cropRect.x(); + rcOutput.top = m_cropRect.y(); + rcOutput.right = m_cropRect.x() + m_cropRect.width(); + rcOutput.bottom = m_cropRect.y() + m_cropRect.height(); + + m_sourceRect.left = float(m_cropRect.x()) / width; + m_sourceRect.top = float(m_cropRect.y()) / height; + m_sourceRect.right = float(m_cropRect.x() + m_cropRect.width()) / width; + m_sourceRect.bottom = float(m_cropRect.y() + m_cropRect.height()) / height; + + if (m_mixer) + configureMixer(m_mixer); + } else { + rcOutput.left = 0; + rcOutput.top = 0; + rcOutput.right = width; + rcOutput.bottom = height; + } // Set the geometric aperture, and disable pan/scan. - displayArea = qt_evr_makeMFArea(0, 0, rcOutput.right, rcOutput.bottom); + displayArea = qt_evr_makeMFArea(0, 0, rcOutput.right - rcOutput.left, + rcOutput.bottom - rcOutput.top); hr = mtOptimal->SetUINT32(MF_MT_PAN_SCAN_ENABLED, FALSE); if (FAILED(hr)) @@ -1427,7 +1451,7 @@ HRESULT EVRCustomPresenter::setMediaType(IMFMediaType *mediaType) // Initialize the presenter engine with the new media type. // The presenter engine allocates the samples. - hr = m_presentEngine->createVideoSamples(mediaType, sampleQueue); + hr = m_presentEngine->createVideoSamples(mediaType, sampleQueue, m_cropRect.size()); if (FAILED(hr)) goto done; diff --git a/src/plugins/common/evr/evrcustompresenter.h b/src/plugins/common/evr/evrcustompresenter.h index c1c21580e..403158619 100644 --- a/src/plugins/common/evr/evrcustompresenter.h +++ b/src/plugins/common/evr/evrcustompresenter.h @@ -44,6 +44,7 @@ #include <qmutex.h> #include <qqueue.h> #include <qevent.h> +#include <qrect.h> #include <qvideosurfaceformat.h> #include "evrdefs.h" @@ -257,6 +258,7 @@ public: void supportedFormatsChanged(); void setSurface(QAbstractVideoSurface *surface); + void setCropRect(QRect cropRect); void startSurface(); void stopSurface(); @@ -368,6 +370,7 @@ private: QAbstractVideoSurface *m_surface; bool m_canRenderToSurface; qint64 m_positionOffset; // Seek position in microseconds. + QRect m_cropRect; // Video crop rectangle }; bool qt_evr_setCustomPresenter(IUnknown *evr, EVRCustomPresenter *presenter); diff --git a/src/plugins/common/evr/evrd3dpresentengine.cpp b/src/plugins/common/evr/evrd3dpresentengine.cpp index d8e2da6d3..0bf8a0506 100644 --- a/src/plugins/common/evr/evrd3dpresentengine.cpp +++ b/src/plugins/common/evr/evrd3dpresentengine.cpp @@ -520,7 +520,7 @@ void D3DPresentEngine::setHint(Hint hint, bool enable) m_useTextureRendering = enable && supportsTextureRendering(); } -HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSample*> &videoSampleQueue) +HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSample*> &videoSampleQueue, QSize frameSize) { if (!format) return MF_E_UNEXPECTED; @@ -537,6 +537,11 @@ HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSamp if (FAILED(hr)) return hr; + if (frameSize.isValid() && !frameSize.isEmpty()) { + width = frameSize.width(); + height = frameSize.height(); + } + DWORD d3dFormat = 0; hr = qt_evr_getFourCC(format, &d3dFormat); if (FAILED(hr)) diff --git a/src/plugins/common/evr/evrd3dpresentengine.h b/src/plugins/common/evr/evrd3dpresentengine.h index 8e2a444f3..d7eeacfc0 100644 --- a/src/plugins/common/evr/evrd3dpresentengine.h +++ b/src/plugins/common/evr/evrd3dpresentengine.h @@ -41,6 +41,7 @@ #define EVRD3DPRESENTENGINE_H #include <QMutex> +#include <QSize> #include <QVideoSurfaceFormat> #include <d3d9.h> @@ -120,7 +121,7 @@ public: bool supportsTextureRendering() const; bool isTextureRenderingEnabled() const { return m_useTextureRendering; } - HRESULT createVideoSamples(IMFMediaType *format, QList<IMFSample*>& videoSampleQueue); + HRESULT createVideoSamples(IMFMediaType *format, QList<IMFSample*>& videoSampleQueue, QSize frameSize); QVideoSurfaceFormat videoSurfaceFormat() const { return m_surfaceFormat; } QVideoFrame makeVideoFrame(IMFSample* sample); diff --git a/src/plugins/common/evr/evrvideowindowcontrol.cpp b/src/plugins/common/evr/evrvideowindowcontrol.cpp index 95f63c2e7..cec263da9 100644 --- a/src/plugins/common/evr/evrvideowindowcontrol.cpp +++ b/src/plugins/common/evr/evrvideowindowcontrol.cpp @@ -134,26 +134,29 @@ void EvrVideoWindowControl::setDisplayRect(const QRect &rect) if (m_displayControl) { RECT displayRect = { rect.left(), rect.top(), rect.right() + 1, rect.bottom() + 1 }; - QSize sourceSize = nativeSize(); + QSize fullSize = nativeSize(); - RECT sourceRect = { 0, 0, sourceSize.width(), sourceSize.height() }; + RECT sourceRect; + if (m_cropRect.isValid()) + sourceRect = { m_cropRect.x(), m_cropRect.y(), m_cropRect.right() + 1, m_cropRect.bottom() + 1 }; + else + sourceRect = { 0, 0, fullSize.width(), fullSize.height() }; if (m_aspectRatioMode == Qt::KeepAspectRatioByExpanding) { QSize clippedSize = rect.size(); clippedSize.scale(sourceRect.right, sourceRect.bottom, Qt::KeepAspectRatio); - sourceRect.left = (sourceRect.right - clippedSize.width()) / 2; sourceRect.top = (sourceRect.bottom - clippedSize.height()) / 2; sourceRect.right = sourceRect.left + clippedSize.width(); sourceRect.bottom = sourceRect.top + clippedSize.height(); } - if (sourceSize.width() > 0 && sourceSize.height() > 0) { + if (sourceRect.right - sourceRect.left > 0 && sourceRect.bottom - sourceRect.top > 0) { MFVideoNormalizedRect sourceNormRect; - sourceNormRect.left = float(sourceRect.left) / float(sourceRect.right); - sourceNormRect.top = float(sourceRect.top) / float(sourceRect.bottom); - sourceNormRect.right = float(sourceRect.right) / float(sourceRect.right); - sourceNormRect.bottom = float(sourceRect.bottom) / float(sourceRect.bottom); + sourceNormRect.left = float(sourceRect.left) / float(fullSize.width()); + sourceNormRect.top = float(sourceRect.top) / float(fullSize.height()); + sourceNormRect.right = float(sourceRect.right) / float(fullSize.width()); + sourceNormRect.bottom = float(sourceRect.bottom) / float(fullSize.height()); m_displayControl->SetVideoPosition(&sourceNormRect, &displayRect); } else { m_displayControl->SetVideoPosition(NULL, &displayRect); @@ -164,6 +167,11 @@ void EvrVideoWindowControl::setDisplayRect(const QRect &rect) } } +void EvrVideoWindowControl::setCropRect(QRect cropRect) +{ + m_cropRect = cropRect; +} + bool EvrVideoWindowControl::isFullScreen() const { return m_fullScreen; diff --git a/src/plugins/common/evr/evrvideowindowcontrol.h b/src/plugins/common/evr/evrvideowindowcontrol.h index ce3b7746f..49d111b99 100644 --- a/src/plugins/common/evr/evrvideowindowcontrol.h +++ b/src/plugins/common/evr/evrvideowindowcontrol.h @@ -85,6 +85,8 @@ public: void applyImageControls(); + void setCropRect(QRect cropRect); + private: void clear(); DXVA2_Fixed32 scaleProcAmpValue(DWORD prop, int value) const; @@ -94,6 +96,7 @@ private: DWORD m_dirtyValues; Qt::AspectRatioMode m_aspectRatioMode; QRect m_displayRect; + QRect m_cropRect; int m_brightness; int m_contrast; int m_hue; diff --git a/src/plugins/gstreamer/camerabin/camerabinsession.cpp b/src/plugins/gstreamer/camerabin/camerabinsession.cpp index 22f5fb116..532fb43ec 100644 --- a/src/plugins/gstreamer/camerabin/camerabinsession.cpp +++ b/src/plugins/gstreamer/camerabin/camerabinsession.cpp @@ -479,6 +479,15 @@ GstElement *CameraBinSession::buildCameraSource() m_inputDeviceHasChanged = false; m_usingWrapperCameraBinSrc = false; + if (m_videoSrc) { + gst_object_unref(GST_OBJECT(m_videoSrc)); + m_videoSrc = 0; + } + if (m_cameraSrc) { + gst_object_unref(GST_OBJECT(m_cameraSrc)); + m_cameraSrc = 0; + } + GstElement *camSrc = 0; g_object_get(G_OBJECT(m_camerabin), CAMERA_SOURCE_PROPERTY, &camSrc, NULL); diff --git a/src/plugins/pulseaudio/qaudioinput_pulse.cpp b/src/plugins/pulseaudio/qaudioinput_pulse.cpp index 2b5325132..b68b4af1b 100644 --- a/src/plugins/pulseaudio/qaudioinput_pulse.cpp +++ b/src/plugins/pulseaudio/qaudioinput_pulse.cpp @@ -402,6 +402,8 @@ int QPulseAudioInput::bytesReady() const qint64 QPulseAudioInput::read(char *data, qint64 len) { + Q_ASSERT(data != nullptr || len == 0); + m_bytesAvailable = checkBytesReady(); setError(QAudio::NoError); @@ -411,7 +413,8 @@ qint64 QPulseAudioInput::read(char *data, qint64 len) if (!m_pullMode && !m_tempBuffer.isEmpty()) { readBytes = qMin(static_cast<int>(len), m_tempBuffer.size()); - memcpy(data, m_tempBuffer.constData(), readBytes); + if (readBytes) + memcpy(data, m_tempBuffer.constData(), readBytes); m_totalTimeValue += readBytes; if (readBytes < m_tempBuffer.size()) { @@ -502,9 +505,10 @@ qint64 QPulseAudioInput::read(char *data, qint64 len) void QPulseAudioInput::applyVolume(const void *src, void *dest, int len) { + Q_ASSERT((src && dest) || len == 0); if (m_volume < 1.f) QAudioHelperInternal::qMultiplySamples(m_volume, m_format, src, dest, len); - else + else if (len) memcpy(dest, src, len); } diff --git a/src/plugins/pulseaudio/qaudiooutput_pulse.cpp b/src/plugins/pulseaudio/qaudiooutput_pulse.cpp index 19ddac1e5..765efe036 100644 --- a/src/plugins/pulseaudio/qaudiooutput_pulse.cpp +++ b/src/plugins/pulseaudio/qaudiooutput_pulse.cpp @@ -506,27 +506,30 @@ qint64 QPulseAudioOutput::write(const char *data, qint64 len) pulseEngine->lock(); - len = qMin(len, static_cast<qint64>(pa_stream_writable_size(m_stream))); + size_t nbytes = len; + void *dest = nullptr; + + if (pa_stream_begin_write(m_stream, &dest, &nbytes) < 0) { + qWarning("QAudioSink(pulseaudio): pa_stream_begin_write, error = %s", + pa_strerror(pa_context_errno(pulseEngine->context()))); + setError(QAudio::IOError); + return 0; + } + + len = qMin(len, qint64(nbytes)); if (m_volume < 1.0f) { // Don't use PulseAudio volume, as it might affect all other streams of the same category // or even affect the system volume if flat volumes are enabled - void *dest = NULL; - size_t nbytes = len; - if (pa_stream_begin_write(m_stream, &dest, &nbytes) < 0) { - qWarning("QAudioOutput(pulseaudio): pa_stream_begin_write, error = %s", - pa_strerror(pa_context_errno(pulseEngine->context()))); - setError(QAudio::IOError); - return 0; - } - - len = int(nbytes); QAudioHelperInternal::qMultiplySamples(m_volume, m_format, data, dest, len); - data = reinterpret_cast<char *>(dest); + } else { + memcpy(dest, data, len); } - if (pa_stream_write(m_stream, data, len, NULL, 0, PA_SEEK_RELATIVE) < 0) { - qWarning("QAudioOutput(pulseaudio): pa_stream_write, error = %s", + data = reinterpret_cast<char *>(dest); + + if ((pa_stream_write(m_stream, data, len, nullptr, 0, PA_SEEK_RELATIVE)) < 0) { + qWarning("QAudioSink(pulseaudio): pa_stream_write, error = %s", pa_strerror(pa_context_errno(pulseEngine->context()))); setError(QAudio::IOError); return 0; @@ -683,7 +686,6 @@ qint64 PulseOutputPrivate::readData(char *data, qint64 len) qint64 PulseOutputPrivate::writeData(const char *data, qint64 len) { - int retry = 0; qint64 written = 0; if ((m_audioDevice->m_deviceState == QAudio::ActiveState @@ -691,10 +693,8 @@ qint64 PulseOutputPrivate::writeData(const char *data, qint64 len) while(written < len) { int chunk = m_audioDevice->write(data+written, (len-written)); if (chunk <= 0) - retry++; - written+=chunk; - if (retry > 10) return written; + written+=chunk; } } diff --git a/src/plugins/qnx-audio/audio/audio.pro b/src/plugins/qnx-audio/audio/audio.pro index bd69dfe1e..d63e58676 100644 --- a/src/plugins/qnx-audio/audio/audio.pro +++ b/src/plugins/qnx-audio/audio/audio.pro @@ -19,4 +19,5 @@ SOURCES += qnxaudioplugin.cpp \ OTHER_FILES += qnx_audio.json PLUGIN_TYPE = audio +PLUGIN_CLASS_NAME = QnxAudioPlugin load(qt_plugin) diff --git a/src/plugins/qnx/common/windowgrabber.cpp b/src/plugins/qnx/common/windowgrabber.cpp index 9a65ad3a0..b2470c250 100644 --- a/src/plugins/qnx/common/windowgrabber.cpp +++ b/src/plugins/qnx/common/windowgrabber.cpp @@ -393,8 +393,12 @@ GLuint WindowGrabberImage::getTexture(screen_window_t window, const QSize &size) { if (size != m_size) { - if (!m_glTexture) - glGenTextures(1, &m_glTexture); + // create a brand new texture to be the KHR image sibling, as + // previously used textures cannot be reused with new KHR image + // sources - note that glDeleteTextures handles nullptr gracefully + glDeleteTextures(1, &m_glTexture); + glGenTextures(1, &m_glTexture); + glBindTexture(GL_TEXTURE_2D, m_glTexture); if (m_eglImage) { glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, 0); diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp index e200e8d16..c44a03896 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp @@ -51,7 +51,9 @@ #include <QtMultimedia/private/qtmultimediaglobal_p.h> #include "private/qgstvideobuffer_p.h" +#if GST_CHECK_VERSION(1,14,0) #include <gst/allocators/gstphysmemory.h> +#endif //#define QT_VIVANTE_VIDEO_DEBUG diff --git a/src/plugins/wmf/player/mfplayersession.cpp b/src/plugins/wmf/player/mfplayersession.cpp index 1d145edc4..daf6f801f 100644 --- a/src/plugins/wmf/player/mfplayersession.cpp +++ b/src/plugins/wmf/player/mfplayersession.cpp @@ -47,6 +47,7 @@ #include <QtCore/qdebug.h> #include <QtCore/qfile.h> #include <QtCore/qbuffer.h> +#include <QtMultimedia/qmediametadata.h> #include "mfplayercontrol.h" #include "mfevrvideowindowcontrol.h" @@ -428,9 +429,15 @@ IMFTopologyNode* MFPlayerSession::addOutputNode(MediaType mediaType, IMFTopology if (mediaType == Audio) { activate = m_playerService->audioEndpointControl()->createActivate(); } else if (mediaType == Video) { + + QSize resolution = m_playerService->metaDataControl()->metaData(QMediaMetaData::Resolution).toSize(); + QRect cropRect = QRect(QPoint(), resolution); + if (m_playerService->videoRendererControl()) { + m_playerService->videoRendererControl()->setCropRect(cropRect); activate = m_playerService->videoRendererControl()->createActivate(); } else if (m_playerService->videoWindowControl()) { + m_playerService->videoWindowControl()->setCropRect(cropRect); activate = m_playerService->videoWindowControl()->createActivate(); } else { qWarning() << "no videoWindowControl or videoRendererControl, unable to add output node for video data"; diff --git a/src/plugins/wmf/player/mfvideorenderercontrol.cpp b/src/plugins/wmf/player/mfvideorenderercontrol.cpp index 94d5f68be..38455090b 100644 --- a/src/plugins/wmf/player/mfvideorenderercontrol.cpp +++ b/src/plugins/wmf/player/mfvideorenderercontrol.cpp @@ -2239,10 +2239,12 @@ public: STDMETHODIMP DetachObject(); void setSurface(QAbstractVideoSurface *surface); + void setCropRect(QRect cropRect); private: EVRCustomPresenter *m_presenter; QAbstractVideoSurface *m_surface; + QRect m_cropRect; QMutex m_mutex; }; @@ -2305,6 +2307,14 @@ void MFVideoRendererControl::setSurface(QAbstractVideoSurface *surface) static_cast<VideoRendererActivate*>(m_currentActivate)->setSurface(m_surface); } +void MFVideoRendererControl::setCropRect(QRect cropRect) +{ + m_cropRect = cropRect; + + if (m_presenterActivate) + m_presenterActivate->setCropRect(cropRect); +} + void MFVideoRendererControl::customEvent(QEvent *event) { if (m_presenterActivate) @@ -2365,6 +2375,7 @@ IMFActivate* MFVideoRendererControl::createActivate() if (SUCCEEDED(MFCreateVideoRendererActivate(::GetShellWindow(), &m_currentActivate))) { m_presenterActivate = new EVRCustomPresenterActivate; m_currentActivate->SetUnknown(MF_ACTIVATE_CUSTOM_VIDEO_PRESENTER_ACTIVATE, m_presenterActivate); + m_presenterActivate->setCropRect(m_cropRect); } else { m_currentActivate = new VideoRendererActivate(this); } @@ -2388,6 +2399,7 @@ HRESULT EVRCustomPresenterActivate::ActivateObject(REFIID riid, void **ppv) QMutexLocker locker(&m_mutex); if (!m_presenter) { m_presenter = new EVRCustomPresenter; + m_presenter->setCropRect(m_cropRect); if (m_surface) m_presenter->setSurface(m_surface); } @@ -2423,5 +2435,17 @@ void EVRCustomPresenterActivate::setSurface(QAbstractVideoSurface *surface) m_presenter->setSurface(surface); } +void EVRCustomPresenterActivate::setCropRect(QRect cropRect) +{ + QMutexLocker locker(&m_mutex); + if (m_cropRect == cropRect) + return; + + m_cropRect = cropRect; + + if (m_presenter) + m_presenter->setCropRect(cropRect); +} + #include "moc_mfvideorenderercontrol.cpp" #include "mfvideorenderercontrol.moc" diff --git a/src/plugins/wmf/player/mfvideorenderercontrol.h b/src/plugins/wmf/player/mfvideorenderercontrol.h index da9e97ba9..8eab19b40 100644 --- a/src/plugins/wmf/player/mfvideorenderercontrol.h +++ b/src/plugins/wmf/player/mfvideorenderercontrol.h @@ -43,6 +43,7 @@ #include "qvideorenderercontrol.h" #include <mfapi.h> #include <mfidl.h> +#include <qrect.h> QT_USE_NAMESPACE @@ -61,6 +62,8 @@ public: IMFActivate* createActivate(); void releaseActivate(); + void setCropRect(QRect cropRect); + protected: void customEvent(QEvent *event); @@ -74,8 +77,8 @@ private: QAbstractVideoSurface *m_surface; IMFActivate *m_currentActivate; IMFSampleGrabberSinkCallback *m_callback; - EVRCustomPresenterActivate *m_presenterActivate; + QRect m_cropRect; }; #endif diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp index ca3a4b2ea..a948a5218 100644 --- a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp +++ b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp @@ -374,6 +374,8 @@ void QDeclarativeVideoOutput::_q_updateNativeSize() if (!m_backend) return; + m_geometryDirty = true; + QSize size = m_backend->nativeSize(); if (!qIsDefaultAspect(m_orientation)) { size.transpose(); @@ -382,8 +384,6 @@ void QDeclarativeVideoOutput::_q_updateNativeSize() if (m_nativeSize != size) { m_nativeSize = size; - m_geometryDirty = true; - setImplicitWidth(size.width()); setImplicitHeight(size.height()); @@ -421,9 +421,8 @@ void QDeclarativeVideoOutput::_q_updateGeometry() } if (m_backend) { - if (!m_backend->videoSurface() || m_backend->videoSurface()->isActive()) - m_backend->updateGeometry(); - else + m_backend->updateGeometry(); + if (m_backend->videoSurface() && !m_backend->videoSurface()->isActive()) m_geometryDirty = true; } diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp index 473a4144f..bac143b43 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp @@ -53,6 +53,13 @@ QList<QVideoFrame::PixelFormat> QSGVideoNodeFactory_Texture::supportedPixelForma { QList<QVideoFrame::PixelFormat> pixelFormats; +#ifdef Q_OS_MACOS + if (handleType == QAbstractVideoBuffer::GLTextureRectangleHandle) { + pixelFormats.append(QVideoFrame::Format_BGR32); + pixelFormats.append(QVideoFrame::Format_BGRA32); + } +#endif + if (handleType == QAbstractVideoBuffer::GLTextureHandle) { pixelFormats.append(QVideoFrame::Format_RGB565); pixelFormats.append(QVideoFrame::Format_RGB32); @@ -82,8 +89,6 @@ public: QSGVideoMaterialShader_Texture() : QSGMaterialShader() { - setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/monoplanarvideo.vert")); - setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag")); } void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; @@ -109,10 +114,20 @@ protected: int m_id_opacity; }; -class QSGVideoMaterialShader_Texture_swizzle : public QSGVideoMaterialShader_Texture +class QSGVideoMaterialShader_Texture_2D : public QSGVideoMaterialShader_Texture { public: - QSGVideoMaterialShader_Texture_swizzle(bool hasAlpha) + QSGVideoMaterialShader_Texture_2D() + { + setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/monoplanarvideo.vert")); + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag")); + } +}; + +class QSGVideoMaterialShader_Texture_2D_swizzle : public QSGVideoMaterialShader_Texture_2D +{ +public: + QSGVideoMaterialShader_Texture_2D_swizzle(bool hasAlpha) : m_hasAlpha(hasAlpha) { setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag")); @@ -120,14 +135,13 @@ public: protected: void initialize() override { - QSGVideoMaterialShader_Texture::initialize(); + QSGVideoMaterialShader_Texture_2D::initialize(); program()->setUniformValue(program()->uniformLocation("hasAlpha"), GLboolean(m_hasAlpha)); } int m_hasAlpha; }; - class QSGVideoMaterial_Texture : public QSGMaterial { public: @@ -149,12 +163,6 @@ public: return needsSwizzling() ? &swizzleType : &normalType; } - QSGMaterialShader *createShader() const override { - const bool hasAlpha = m_format.pixelFormat() == QVideoFrame::Format_ARGB32; - return needsSwizzling() ? new QSGVideoMaterialShader_Texture_swizzle(hasAlpha) - : new QSGVideoMaterialShader_Texture; - } - int compare(const QSGMaterial *other) const override { const QSGVideoMaterial_Texture *m = static_cast<const QSGVideoMaterial_Texture *>(other); @@ -179,9 +187,42 @@ public: void setVideoFrame(const QVideoFrame &frame) { QMutexLocker lock(&m_frameMutex); m_frame = frame; + m_textureSize = frame.size(); + } + + virtual void bind() = 0; + + QVideoFrame m_frame; + QMutex m_frameMutex; + QSize m_textureSize; + QVideoSurfaceFormat m_format; + GLuint m_textureId; + qreal m_opacity; + +protected: + bool needsSwizzling() const { + return !QMediaOpenGLHelper::isANGLE() + && (m_format.pixelFormat() == QVideoFrame::Format_RGB32 + || m_format.pixelFormat() == QVideoFrame::Format_ARGB32); + } +}; + +class QSGVideoMaterial_Texture_2D : public QSGVideoMaterial_Texture +{ +public: + QSGVideoMaterial_Texture_2D(const QVideoSurfaceFormat &format) : + QSGVideoMaterial_Texture(format) + { } - void bind() + QSGMaterialShader *createShader() const override + { + const bool hasAlpha = m_format.pixelFormat() == QVideoFrame::Format_ARGB32; + return needsSwizzling() ? new QSGVideoMaterialShader_Texture_2D_swizzle(hasAlpha) + : new QSGVideoMaterialShader_Texture_2D; + } + + void bind() override { QMutexLocker lock(&m_frameMutex); if (m_frame.isValid()) { @@ -197,28 +238,84 @@ public: m_textureId = 0; } } +}; + +#ifdef Q_OS_MACOS +class QSGVideoMaterialShader_Texture_Rectangle : public QSGVideoMaterialShader_Texture +{ +public: + QSGVideoMaterialShader_Texture_Rectangle() + { + setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/rectsampler.vert")); + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rectsampler_rgb.frag")); + } - QVideoFrame m_frame; - QMutex m_frameMutex; - QSize m_textureSize; - QVideoSurfaceFormat m_format; - GLuint m_textureId; - qreal m_opacity; + void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override + { + QSGVideoMaterial_Texture *mat = static_cast<QSGVideoMaterial_Texture *>(newMaterial); + QVector2D size(mat->m_textureSize.width(), mat->m_textureSize.height()); + program()->setUniformValue(m_id_videoSize, size); -private: - bool needsSwizzling() const { - return !QMediaOpenGLHelper::isANGLE() - && (m_format.pixelFormat() == QVideoFrame::Format_RGB32 - || m_format.pixelFormat() == QVideoFrame::Format_ARGB32); + QSGVideoMaterialShader_Texture::updateState(state, newMaterial, oldMaterial); } + +protected: + void initialize() override + { + QSGVideoMaterialShader_Texture::initialize(); + m_id_videoSize = program()->uniformLocation("qt_videoSize"); + } + + int m_id_videoSize; }; +class QSGVideoMaterial_Texture_Rectangle : public QSGVideoMaterial_Texture +{ +public: + QSGVideoMaterial_Texture_Rectangle(const QVideoSurfaceFormat &format) : + QSGVideoMaterial_Texture(format) + { + } + + QSGMaterialShader *createShader() const override + { + Q_ASSERT(!needsSwizzling()); + return new QSGVideoMaterialShader_Texture_Rectangle; + } + + void bind() override + { + QMutexLocker lock(&m_frameMutex); + if (m_frame.isValid()) { + m_textureId = m_frame.handle().toUInt(); + QOpenGLFunctions *functions = QOpenGLContext::currentContext()->functions(); + functions->glActiveTexture(GL_TEXTURE0); + functions->glBindTexture(GL_TEXTURE_RECTANGLE, m_textureId); + + functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } else { + m_textureId = 0; + } + } +}; +#endif QSGVideoNode_Texture::QSGVideoNode_Texture(const QVideoSurfaceFormat &format) : m_format(format) { setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_Texture(format); + +#ifdef Q_OS_MACOS + if (format.handleType() == QAbstractVideoBuffer::GLTextureRectangleHandle) + m_material = new QSGVideoMaterial_Texture_Rectangle(format); +#endif + + if (!m_material) + m_material = new QSGVideoMaterial_Texture_2D(format); + setMaterial(m_material); } diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h b/src/qtmultimediaquicktools/qsgvideonode_texture_p.h index 12685dd24..21c905bd5 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h +++ b/src/qtmultimediaquicktools/qsgvideonode_texture_p.h @@ -74,7 +74,7 @@ public: private: QVideoSurfaceFormat m_format; - QSGVideoMaterial_Texture *m_material; + QSGVideoMaterial_Texture *m_material = nullptr; QVideoFrame m_frame; }; diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc index b8180e31f..86523e771 100644 --- a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc +++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc @@ -11,7 +11,6 @@ <file>shaders/triplanaryuvvideo.vert</file> <file>shaders/uyvyvideo.frag</file> <file>shaders/yuyvvideo.frag</file> - <file>shaders/monoplanarvideo_core.vert</file> <file>shaders/rgbvideo_core.frag</file> <file>shaders/rgbvideo_swizzle_core.frag</file> @@ -23,5 +22,9 @@ <file>shaders/triplanaryuvvideo_core.vert</file> <file>shaders/uyvyvideo_core.frag</file> <file>shaders/yuyvvideo_core.frag</file> + <file>shaders/rectsampler.vert</file> + <file>shaders/rectsampler_rgb.frag</file> + <file>shaders/rectsampler_core.vert</file> + <file>shaders/rectsampler_rgb_core.frag</file> </qresource> </RCC> diff --git a/src/qtmultimediaquicktools/shaders/rectsampler.vert b/src/qtmultimediaquicktools/shaders/rectsampler.vert new file mode 100644 index 000000000..762ec7e7e --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rectsampler.vert @@ -0,0 +1,10 @@ +uniform highp mat4 qt_Matrix; +uniform highp vec2 qt_videoSize; +attribute highp vec4 qt_VertexPosition; +attribute highp vec2 qt_VertexTexCoord; +varying highp vec2 qt_TexCoord; + +void main() { + qt_TexCoord = vec2(qt_VertexTexCoord.x * qt_videoSize.x, qt_VertexTexCoord.y * qt_videoSize.y); + gl_Position = qt_Matrix * qt_VertexPosition; +} diff --git a/src/qtmultimediaquicktools/shaders/rectsampler_core.vert b/src/qtmultimediaquicktools/shaders/rectsampler_core.vert new file mode 100644 index 000000000..f0fe02349 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rectsampler_core.vert @@ -0,0 +1,11 @@ +#version 150 core +uniform highp mat4 qt_Matrix; +uniform highp vec2 qt_videoSize; +in highp vec4 qt_VertexPosition; +in highp vec2 qt_VertexTexCoord; +out highp vec2 qt_TexCoord; + +void main() { + qt_TexCoord = vec2(qt_VertexTexCoord.x * qt_videoSize.x, qt_VertexTexCoord.y * qt_videoSize.y); + gl_Position = qt_Matrix * qt_VertexPosition; +} diff --git a/src/qtmultimediaquicktools/shaders/rectsampler_rgb.frag b/src/qtmultimediaquicktools/shaders/rectsampler_rgb.frag new file mode 100644 index 000000000..2a30f7c3d --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rectsampler_rgb.frag @@ -0,0 +1,8 @@ +uniform sampler2DRect rgbTexture; +uniform lowp float opacity; +varying highp vec2 qt_TexCoord; + +void main() +{ + gl_FragColor = texture2DRect(rgbTexture, qt_TexCoord) * opacity; +} diff --git a/src/qtmultimediaquicktools/shaders/rectsampler_rgb_core.frag b/src/qtmultimediaquicktools/shaders/rectsampler_rgb_core.frag new file mode 100644 index 000000000..17f306456 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rectsampler_rgb_core.frag @@ -0,0 +1,10 @@ +#version 150 core +uniform sampler2DRect rgbTexture; +uniform lowp float opacity; +in highp vec2 qt_TexCoord; +out vec4 fragColor; + +void main() +{ + fragColor = texture(rgbTexture, qt_TexCoord) * opacity; +} |