diff options
Diffstat (limited to 'src/multimedia/video/qvideoframeconverter.cpp')
| -rw-r--r-- | src/multimedia/video/qvideoframeconverter.cpp | 190 |
1 files changed, 96 insertions, 94 deletions
diff --git a/src/multimedia/video/qvideoframeconverter.cpp b/src/multimedia/video/qvideoframeconverter.cpp index 6ad1e8dc2..ad1e96d79 100644 --- a/src/multimedia/video/qvideoframeconverter.cpp +++ b/src/multimedia/video/qvideoframeconverter.cpp @@ -1,60 +1,11 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qvideoframeconverter_p.h" #include "qvideoframeconversionhelper_p.h" #include "qvideoframeformat.h" - -#include <QtGui/private/qrhinull_p.h> -#if QT_CONFIG(opengl) -#include <QtGui/private/qrhigles2_p.h> -#include <QOffscreenSurface> -#endif -#if QT_CONFIG(vulkan) -#include <QtGui/private/qrhivulkan_p.h> -#endif -#ifdef Q_OS_WIN -#include <QtGui/private/qrhid3d11_p.h> -#endif -#if defined(Q_OS_MACOS) || defined(Q_OS_IOS) -#include <QtGui/private/qrhimetal_p.h> -#endif +#include "qvideoframe_p.h" +#include "qmultimediautils_p.h" #include <QtCore/qcoreapplication.h> #include <QtCore/qsize.h> @@ -62,16 +13,20 @@ #include <QtCore/qfile.h> #include <QtCore/qthreadstorage.h> #include <QtGui/qimage.h> +#include <QtGui/qoffscreensurface.h> #include <qpa/qplatformintegration.h> #include <private/qvideotexturehelper_p.h> #include <private/qabstractvideobuffer_p.h> #include <private/qguiapplication_p.h> -#include <private/qrhi_p.h> +#include <rhi/qrhi.h> +#ifdef Q_OS_DARWIN +#include <QtCore/private/qcore_mac_p.h> +#endif QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcVideoFrameConverter, "qt.multimedia.video.frameconverter") +static Q_LOGGING_CATEGORY(qLcVideoFrameConverter, "qt.multimedia.video.frameconverter") namespace { @@ -82,11 +37,21 @@ struct State QOffscreenSurface *fallbackSurface = nullptr; #endif bool cpuOnly = false; +#if defined(Q_OS_ANDROID) + QMetaObject::Connection appStateChangedConnection; +#endif ~State() { + resetRhi(); + } + + void resetRhi() { delete rhi; + rhi = nullptr; #if QT_CONFIG(opengl) delete fallbackSurface; + fallbackSurface = nullptr; #endif + cpuOnly = false; } }; @@ -135,7 +100,7 @@ static bool pixelFormatHasAlpha(QVideoFrameFormat::PixelFormat format) } }; -static QShader getShader(const QString &name) +static QShader vfcGetShader(const QString &name) { QShader shader = g_shaderCache.value(name); if (shader.isValid()) @@ -151,13 +116,13 @@ static QShader getShader(const QString &name) return shader; } -static void rasterTransform(QImage &image, QVideoFrame::RotationAngle rotation, +static void rasterTransform(QImage &image, QtVideo::Rotation rotation, bool mirrorX, bool mirrorY) { QTransform t; if (mirrorX) t.scale(-1.f, 1.f); - if (rotation != QVideoFrame::Rotation0) + if (rotation != QtVideo::Rotation::None) t.rotate(float(rotation)); if (mirrorY) t.scale(1.f, -1.f); @@ -171,11 +136,13 @@ static void imageCleanupHandler(void *info) delete imageData; } -static QRhi *initializeRHI(QRhi::Implementation backend) +static QRhi *initializeRHI(QRhi *videoFrameRhi) { if (g_state.localData().rhi || g_state.localData().cpuOnly) return g_state.localData().rhi; + QRhi::Implementation backend = videoFrameRhi ? videoFrameRhi->backend() : QRhi::Null; + if (QGuiApplicationPrivate::platformIntegration()->hasCapability(QPlatformIntegration::RhiBasedRendering)) { #if defined(Q_OS_MACOS) || defined(Q_OS_IOS) @@ -201,7 +168,19 @@ static QRhi *initializeRHI(QRhi::Implementation backend) g_state.localData().fallbackSurface = QRhiGles2InitParams::newFallbackSurface(); QRhiGles2InitParams params; params.fallbackSurface = g_state.localData().fallbackSurface; + if (backend == QRhi::OpenGLES2) + params.shareContext = static_cast<const QRhiGles2NativeHandles*>(videoFrameRhi->nativeHandles())->context; g_state.localData().rhi = QRhi::create(QRhi::OpenGLES2, ¶ms); + +#if defined(Q_OS_ANDROID) + // reset RHI state on application suspension, as this will be invalid after resuming + if (!g_state.localData().appStateChangedConnection) { + g_state.localData().appStateChangedConnection = QObject::connect(qApp, &QGuiApplication::applicationStateChanged, qApp, [](auto state) { + if (state == Qt::ApplicationSuspended) + g_state.localData().resetRhi(); + }); + } +#endif } } #endif @@ -216,41 +195,37 @@ static QRhi *initializeRHI(QRhi::Implementation backend) } static bool updateTextures(QRhi *rhi, - QRhiResourceUpdateBatch *rub, std::unique_ptr<QRhiBuffer> &uniformBuffer, std::unique_ptr<QRhiSampler> &textureSampler, std::unique_ptr<QRhiShaderResourceBindings> &shaderResourceBindings, std::unique_ptr<QRhiGraphicsPipeline> &graphicsPipeline, std::unique_ptr<QRhiRenderPassDescriptor> &renderPass, - const QVideoFrame &frame, - std::unique_ptr<QRhiTexture> (&textures)[QVideoTextureHelper::TextureDescription::maxPlanes]) + QVideoFrame &frame, + const std::unique_ptr<QVideoFrameTextures> &videoFrameTextures) { auto format = frame.surfaceFormat(); auto pixelFormat = format.pixelFormat(); auto textureDesc = QVideoTextureHelper::textureDescription(pixelFormat); - for (int i = 0; i < QVideoTextureHelper::TextureDescription::maxPlanes; ++i) - QVideoTextureHelper::updateRhiTexture(frame, rhi, rub, i, textures[i]); - QRhiShaderResourceBinding bindings[4]; auto *b = bindings; *b++ = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage, uniformBuffer.get()); for (int i = 0; i < textureDesc->nplanes; ++i) *b++ = QRhiShaderResourceBinding::sampledTexture(i + 1, QRhiShaderResourceBinding::FragmentStage, - textures[i].get(), textureSampler.get()); + videoFrameTextures->texture(i), textureSampler.get()); shaderResourceBindings->setBindings(bindings, b); shaderResourceBindings->create(); graphicsPipeline.reset(rhi->newGraphicsPipeline()); graphicsPipeline->setTopology(QRhiGraphicsPipeline::TriangleStrip); - QShader vs = getShader(QVideoTextureHelper::vertexShaderFileName(format)); + QShader vs = vfcGetShader(QVideoTextureHelper::vertexShaderFileName(format)); if (!vs.isValid()) return false; - QShader fs = getShader(QVideoTextureHelper::fragmentShaderFileName(format)); + QShader fs = vfcGetShader(QVideoTextureHelper::fragmentShaderFileName(format)); if (!fs.isValid()) return false; @@ -276,7 +251,7 @@ static bool updateTextures(QRhi *rhi, return true; } -static QImage convertJPEG(const QVideoFrame &frame, QVideoFrame::RotationAngle rotation, bool mirrorX, bool mirrorY) +static QImage convertJPEG(const QVideoFrame &frame, QtVideo::Rotation rotation, bool mirrorX, bool mirrorY) { QVideoFrame varFrame = frame; if (!varFrame.map(QVideoFrame::ReadOnly)) { @@ -290,7 +265,7 @@ static QImage convertJPEG(const QVideoFrame &frame, QVideoFrame::RotationAngle r return image; } -static QImage convertCPU(const QVideoFrame &frame, QVideoFrame::RotationAngle rotation, bool mirrorX, bool mirrorY) +static QImage convertCPU(const QVideoFrame &frame, QtVideo::Rotation rotation, bool mirrorX, bool mirrorY) { VideoFrameConvertFunc convert = qConverterForFormat(frame.pixelFormat()); if (!convert) { @@ -311,7 +286,8 @@ static QImage convertCPU(const QVideoFrame &frame, QVideoFrame::RotationAngle ro } } -QImage qImageFromVideoFrame(const QVideoFrame &frame, QVideoFrame::RotationAngle rotation, bool mirrorX, bool mirrorY) +QImage qImageFromVideoFrame(const QVideoFrame &frame, QtVideo::Rotation rotation, bool mirrorX, + bool mirrorY, bool forceCpu) { #ifdef Q_OS_DARWIN QMacAutoReleasePool releasePool; @@ -328,7 +304,6 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QVideoFrame::RotationAngle std::unique_ptr<QRhiSampler> textureSampler; std::unique_ptr<QRhiShaderResourceBindings> shaderResourceBindings; std::unique_ptr<QRhiGraphicsPipeline> graphicsPipeline; - std::unique_ptr<QRhiTexture> frameTextures[QVideoTextureHelper::TextureDescription::maxPlanes]; if (frame.size().isEmpty() || frame.pixelFormat() == QVideoFrameFormat::Format_Invalid) return {}; @@ -336,28 +311,23 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QVideoFrame::RotationAngle if (frame.pixelFormat() == QVideoFrameFormat::Format_Jpeg) return convertJPEG(frame, rotation, mirrorX, mirrorY); + if (forceCpu) // For test purposes + return convertCPU(frame, rotation, mirrorX, mirrorY); + QRhi *rhi = nullptr; - QRhi::Implementation backend = QRhi::Null; - if (frame.videoBuffer()) { + if (frame.videoBuffer()) rhi = frame.videoBuffer()->rhi(); - if (rhi) - backend = rhi->backend(); - } if (!rhi || rhi->thread() != QThread::currentThread()) - rhi = initializeRHI(backend); + rhi = initializeRHI(rhi); if (!rhi || rhi->isRecordingFrame()) return convertCPU(frame, rotation, mirrorX, mirrorY); // Do conversion using shaders - const int rotationIndex = (rotation / 90) % 4; - - QSize frameSize = frame.size(); - if (rotationIndex % 2) - frameSize.transpose(); + const QSize frameSize = qRotatedFrameSize(frame.size(), rotation); vertexBuffer.reset(rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_quad))); vertexBuffer->create(); @@ -393,8 +363,15 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QVideoFrame::RotationAngle rub->uploadStaticBuffer(vertexBuffer.get(), g_quad); - if (!updateTextures(rhi, rub, uniformBuffer, textureSampler, shaderResourceBindings, - graphicsPipeline, renderPass, frame, frameTextures)) { + QVideoFrame frameTmp = frame; + auto videoFrameTextures = QVideoTextureHelper::createTextures(frameTmp, rhi, rub, {}); + if (!videoFrameTextures) { + qCDebug(qLcVideoFrameConverter) << "Failed obtain textures. Using CPU conversion."; + return convertCPU(frame, rotation, mirrorX, mirrorY); + } + + if (!updateTextures(rhi, uniformBuffer, textureSampler, shaderResourceBindings, + graphicsPipeline, renderPass, frameTmp, videoFrameTextures)) { qCDebug(qLcVideoFrameConverter) << "Failed to update textures. Using CPU conversion."; return convertCPU(frame, rotation, mirrorX, mirrorY); } @@ -418,7 +395,8 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QVideoFrame::RotationAngle cb->setViewport({ 0, 0, float(frameSize.width()), float(frameSize.height()) }); cb->setShaderResources(shaderResourceBindings.get()); - quint32 vertexOffset = quint32(sizeof(float)) * 16 * rotationIndex; + const int rotationIndex = (qToUnderlying(rotation) / 90) % 4; + const quint32 vertexOffset = quint32(sizeof(float)) * 16 * rotationIndex; const QRhiCommandBuffer::VertexInput vbufBinding(vertexBuffer.get(), vertexOffset); cb->setVertexInput(0, 1, &vbufBinding); cb->draw(4); @@ -441,19 +419,43 @@ QImage qImageFromVideoFrame(const QVideoFrame &frame, QVideoFrame::RotationAngle return convertCPU(frame, rotation, mirrorX, mirrorY); } - if (!qConverterForFormat(frame.pixelFormat())) { - qCDebug(qLcVideoFrameConverter) << "Unsupported pixel format" << frame.pixelFormat(); + QByteArray *imageData = new QByteArray(readResult.data); + + return QImage(reinterpret_cast<const uchar *>(imageData->constData()), + readResult.pixelSize.width(), readResult.pixelSize.height(), + QImage::Format_RGBA8888_Premultiplied, imageCleanupHandler, imageData); +} + +QImage videoFramePlaneAsImage(QVideoFrame &frame, int plane, QImage::Format targetFormat, + QSize targetSize) +{ + if (plane >= frame.planeCount()) + return {}; + + if (!frame.map(QVideoFrame::ReadOnly)) { + qWarning() << "Cannot map a video frame in ReadOnly mode!"; return {}; } - QImage::Format format = pixelFormatHasAlpha(frame.pixelFormat()) ? - QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32; + auto frameHandle = QVideoFramePrivate::handle(frame); - QByteArray *imageData = new QByteArray(readResult.data); + // With incrementing the reference counter, we share the mapped QVideoFrame + // with the target QImage. The function imageCleanupFunction is going to adopt + // the frameHandle by QVideoFrame and dereference it upon the destruction. + frameHandle->ref.ref(); - return QImage(reinterpret_cast<const uchar *>(imageData->constData()), - readResult.pixelSize.width(), readResult.pixelSize.height(), - format, imageCleanupHandler, imageData); + auto imageCleanupFunction = [](void *data) { + QVideoFrame frame = reinterpret_cast<QVideoFramePrivate *>(data)->adoptThisByVideoFrame(); + Q_ASSERT(frame.isMapped()); + frame.unmap(); + }; + + const auto bytesPerLine = frame.bytesPerLine(plane); + const auto height = + bytesPerLine ? qMin(targetSize.height(), frame.mappedBytes(plane) / bytesPerLine) : 0; + + return QImage(reinterpret_cast<const uchar *>(frame.bits(plane)), targetSize.width(), height, + bytesPerLine, targetFormat, imageCleanupFunction, frameHandle); } QT_END_NAMESPACE |