diff options
Diffstat (limited to 'src')
18 files changed, 734 insertions, 1363 deletions
diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt index ce8f44f61..8a1010188 100644 --- a/src/multimedia/CMakeLists.txt +++ b/src/multimedia/CMakeLists.txt @@ -65,6 +65,7 @@ qt_internal_add_module(Multimedia video/qmemoryvideobuffer.cpp video/qmemoryvideobuffer_p.h video/qvideoframe.cpp video/qvideoframe.h video/qvideosink.cpp video/qvideosink.h + video/qvideotexturehelper.cpp video/qvideotexturehelper_p.h video/qvideoframeconversionhelper.cpp video/qvideoframeconversionhelper_p.h video/qvideooutputorientationhandler.cpp video/qvideooutputorientationhandler_p.h video/qvideosurfaceformat.cpp video/qvideosurfaceformat.h diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp index 2fa5f9f88..a4a985a5f 100644 --- a/src/multimedia/video/qvideoframe.cpp +++ b/src/multimedia/video/qvideoframe.cpp @@ -49,6 +49,7 @@ #include <qpair.h> #include <qsize.h> #include <qvariant.h> +#include <private/qrhi_p.h> #include <QDebug> @@ -812,7 +813,7 @@ int QVideoFrame::planeCount() const \internal Returns a texture id to the video frame's buffers. */ -quint64 QVideoFrame::textureHandle(int plane) +quint64 QVideoFrame::textureHandle(int plane) const { return d->buffer->textureHandle(plane); } diff --git a/src/multimedia/video/qvideoframe.h b/src/multimedia/video/qvideoframe.h index ea01ccb92..d2c371c3f 100644 --- a/src/multimedia/video/qvideoframe.h +++ b/src/multimedia/video/qvideoframe.h @@ -53,6 +53,9 @@ QT_BEGIN_NAMESPACE class QSize; class QVideoFramePrivate; class QAbstractVideoBuffer; +class QRhi; +class QRhiResourceUpdateBatch; +class QRhiTexture; class Q_MULTIMEDIA_EXPORT QVideoFrame { @@ -113,7 +116,7 @@ public: int mappedBytes() const; int planeCount() const; - quint64 textureHandle(int plane); + quint64 textureHandle(int plane) const; qint64 startTime() const; void setStartTime(qint64 time); diff --git a/src/multimedia/video/qvideosurfaceformat.cpp b/src/multimedia/video/qvideosurfaceformat.cpp index 5271a12db..745fe8b32 100644 --- a/src/multimedia/video/qvideosurfaceformat.cpp +++ b/src/multimedia/video/qvideosurfaceformat.cpp @@ -38,6 +38,7 @@ ****************************************************************************/ #include "qvideosurfaceformat.h" +#include "qvideotexturehelper_p.h" #include <qdebug.h> #include <qlist.h> @@ -259,49 +260,7 @@ int QVideoSurfaceFormat::frameHeight() const int QVideoSurfaceFormat::nPlanes() const { - switch (d->pixelFormat) { - case Format_Invalid: - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - case Format_UYVY: - case Format_YUYV: - case Format_Y8: - case Format_Y16: - case Format_Jpeg: - return 1; - case Format_NV12: - case Format_NV21: - case Format_IMC2: - case Format_IMC4: - case Format_P010LE: - case Format_P010BE: - case Format_P016LE: - case Format_P016BE: - return 2; - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: - case Format_IMC1: - case Format_IMC3: - return 3; - } - return 0; + return QVideoTextureHelper::textureDescription(d->pixelFormat)->nplanes; } /*! @@ -438,211 +397,17 @@ QSize QVideoSurfaceFormat::sizeHint() const QString QVideoSurfaceFormat::vertexShaderFileName() const { - switch (d->pixelFormat) { - case Format_Invalid: - case Format_Jpeg: - - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - - case Format_Y8: - case Format_Y16: - - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - - case Format_IMC1: - case Format_IMC2: - case Format_IMC3: - case Format_IMC4: - return QString(); - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: - return QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb"); - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: - case Format_UYVY: - case Format_YUYV: - case Format_NV12: - case Format_NV21: - case Format_P010LE: - case Format_P010BE: - case Format_P016LE: - case Format_P016BE: - return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb"); - } + return QVideoTextureHelper::vertexShaderFileName(d->pixelFormat); } QString QVideoSurfaceFormat::fragmentShaderFileName() const { - switch (d->pixelFormat) { - case Format_Invalid: - case Format_Jpeg: - - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - - case Format_Y8: - case Format_Y16: - - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - - case Format_IMC1: - case Format_IMC2: - case Format_IMC3: - case Format_IMC4: - return QString(); - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: - return QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb"); - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: - return QStringLiteral(":/qtmultimedia/shaders/yuv_yv.frag.qsb"); - case Format_UYVY: - return QStringLiteral(":/qtmultimedia/shaders/uyvy.frag.qsb"); - case Format_YUYV: - return QStringLiteral(":/qtmultimedia/shaders/yuyv.frag.qsb"); - case Format_NV12: - return QStringLiteral(":/qtmultimedia/shaders/nv12.frag.qsb"); - case Format_NV21: - return QStringLiteral(":/qtmultimedia/shaders/nv21.frag.qsb"); - case Format_P010LE: - case Format_P016LE: - return QStringLiteral(":/qtmultimedia/shaders/p010le.frag.qsb"); - case Format_P010BE: - case Format_P016BE: - return QStringLiteral(":/qtmultimedia/shaders/p010be.frag.qsb"); - } -} - -static QMatrix4x4 colorMatrix(QVideoSurfaceFormat::YCbCrColorSpace colorSpace) -{ - switch (colorSpace) { - case QVideoSurfaceFormat::YCbCr_JPEG: - return QMatrix4x4( - 1.0f, 0.000f, 1.402f, -0.701f, - 1.0f, -0.344f, -0.714f, 0.529f, - 1.0f, 1.772f, 0.000f, -0.886f, - 0.0f, 0.000f, 0.000f, 1.0000f); - case QVideoSurfaceFormat::YCbCr_BT709: - case QVideoSurfaceFormat::YCbCr_xvYCC709: - return QMatrix4x4( - 1.164f, 0.000f, 1.793f, -0.5727f, - 1.164f, -0.534f, -0.213f, 0.3007f, - 1.164f, 2.115f, 0.000f, -1.1302f, - 0.0f, 0.000f, 0.000f, 1.0000f); - default: //BT 601: - return QMatrix4x4( - 1.164f, 0.000f, 1.596f, -0.8708f, - 1.164f, -0.392f, -0.813f, 0.5296f, - 1.164f, 2.017f, 0.000f, -1.081f, - 0.0f, 0.000f, 0.000f, 1.0000f); - } + return QVideoTextureHelper::fragmentShaderFileName(d->pixelFormat); } QByteArray QVideoSurfaceFormat::uniformData(const QMatrix4x4 &transform, float opacity) const { - static constexpr float pw[3] = {}; - const float *planeWidth = pw; - - switch (d->pixelFormat) { - case Format_Invalid: - case Format_Jpeg: - - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - - case Format_Y8: - case Format_Y16: - - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - - case Format_IMC1: - case Format_IMC2: - case Format_IMC3: - case Format_IMC4: - return QByteArray(); - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: { - // { matrix4x4, opacity } - QByteArray buf(16*4 + 4, Qt::Uninitialized); - char *data = buf.data(); - memcpy(data, transform.constData(), 64); - memcpy(data + 64, &opacity, 4); - return buf; - } - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: { - static constexpr float pw[] = { 1, 1, 1 }; - planeWidth = pw; - break; - } - case Format_UYVY: - case Format_YUYV: { - static constexpr float pw[] = { 1, 1, 0 }; - planeWidth = pw; - break; - } - case Format_NV12: - case Format_NV21: - case Format_P010LE: - case Format_P010BE: - case Format_P016LE: - case Format_P016BE: { - static constexpr float pw[] = { 1, 1, 0 }; - planeWidth = pw; - break; - } - } - // { matrix4x4, colorMatrix, opacity, planeWidth[3] } - QByteArray buf(64*2 + 4 + 3*4, Qt::Uninitialized); - char *data = buf.data(); - memcpy(data, transform.constData(), 64); - memcpy(data + 64, colorMatrix(d->ycbcrColorSpace).constData(), 64); - memcpy(data + 64 + 64, &opacity, 4); - memcpy(data + 64 + 64 + 4, planeWidth, 3*4); - return buf; + return QVideoTextureHelper::uniformData(*this, transform, opacity); } diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp new file mode 100644 index 000000000..930456d7d --- /dev/null +++ b/src/multimedia/video/qvideotexturehelper.cpp @@ -0,0 +1,514 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qvideotexturehelper_p.h" +#include "qvideoframe.h" + +namespace QVideoTextureHelper +{ + +static const TextureDescription descriptions[QVideoSurfaceFormat::NPixelFormats] = { + // Format_Invalid + { 0, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat}, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ARGB32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ARGB32_Premultiplied + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB24 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB565 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB555 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ARGB8565_Premultiplied + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGRA32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGRA32_Premultiplied + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ABGR32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR24 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR565 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR555 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGRA5658_Premultiplied + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + + // Format_AYUV444 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_AYUV444_Premultiplied + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_YUV444 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_YUV420P + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 2, 2 }, { 2, 2 } } + }, + // Format_YUV422P + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 2, 1 }, { 2, 1 } } + }, + // Format_YV12 + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 2, 2 }, { 2, 2 } } + }, + // Format_UYVY + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_YUYV + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_NV12 + { 2, + { QRhiTexture::RG8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_NV21 + { 2, + { QRhiTexture::RG8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_IMC1 + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_IMC2 + { 2, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_IMC3 + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_IMC4 + { 2, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_Y8 + { 1, + { QRhiTexture::R8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_Y16 + { 1, + { QRhiTexture::R16, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + + // Format_P010LE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_P010BE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_P016LE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_P016BE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + + // Format_Jpeg + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + } +}; + + +const TextureDescription *textureDescription(QVideoSurfaceFormat::PixelFormat format) +{ + return descriptions + format; +} + +QString vertexShaderFileName(QVideoSurfaceFormat::PixelFormat format) +{ + switch (format) { + case QVideoSurfaceFormat::Format_Invalid: + case QVideoSurfaceFormat::Format_Jpeg: + + case QVideoSurfaceFormat::Format_RGB24: + case QVideoSurfaceFormat::Format_RGB565: + case QVideoSurfaceFormat::Format_RGB555: + case QVideoSurfaceFormat::Format_ARGB8565_Premultiplied: + case QVideoSurfaceFormat::Format_BGR24: + case QVideoSurfaceFormat::Format_BGR565: + case QVideoSurfaceFormat::Format_BGR555: + case QVideoSurfaceFormat::Format_BGRA5658_Premultiplied: + + case QVideoSurfaceFormat::Format_Y8: + case QVideoSurfaceFormat::Format_Y16: + + case QVideoSurfaceFormat::Format_YUV444: + + case QVideoSurfaceFormat::Format_IMC1: + case QVideoSurfaceFormat::Format_IMC2: + case QVideoSurfaceFormat::Format_IMC3: + case QVideoSurfaceFormat::Format_IMC4: + return QString(); + case QVideoSurfaceFormat::Format_AYUV444: + case QVideoSurfaceFormat::Format_AYUV444_Premultiplied: + return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb"); + case QVideoSurfaceFormat::Format_ARGB32: + case QVideoSurfaceFormat::Format_ARGB32_Premultiplied: + case QVideoSurfaceFormat::Format_RGB32: + case QVideoSurfaceFormat::Format_BGRA32: + case QVideoSurfaceFormat::Format_BGRA32_Premultiplied: + case QVideoSurfaceFormat::Format_ABGR32: + case QVideoSurfaceFormat::Format_BGR32: + return QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb"); + case QVideoSurfaceFormat::Format_YUV420P: + case QVideoSurfaceFormat::Format_YUV422P: + case QVideoSurfaceFormat::Format_YV12: + case QVideoSurfaceFormat::Format_UYVY: + case QVideoSurfaceFormat::Format_YUYV: + case QVideoSurfaceFormat::Format_NV12: + case QVideoSurfaceFormat::Format_NV21: + case QVideoSurfaceFormat::Format_P010LE: + case QVideoSurfaceFormat::Format_P010BE: + case QVideoSurfaceFormat::Format_P016LE: + case QVideoSurfaceFormat::Format_P016BE: + return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb"); + } +} + +QString fragmentShaderFileName(QVideoSurfaceFormat::PixelFormat format) +{ + switch (format) { + case QVideoSurfaceFormat::Format_Invalid: + case QVideoSurfaceFormat::Format_Jpeg: + + case QVideoSurfaceFormat::Format_RGB24: + case QVideoSurfaceFormat::Format_RGB565: + case QVideoSurfaceFormat::Format_RGB555: + case QVideoSurfaceFormat::Format_ARGB8565_Premultiplied: + case QVideoSurfaceFormat::Format_BGR24: + case QVideoSurfaceFormat::Format_BGR565: + case QVideoSurfaceFormat::Format_BGR555: + case QVideoSurfaceFormat::Format_BGRA5658_Premultiplied: + + case QVideoSurfaceFormat::Format_Y8: + case QVideoSurfaceFormat::Format_Y16: + + case QVideoSurfaceFormat::Format_YUV444: + + case QVideoSurfaceFormat::Format_IMC1: + case QVideoSurfaceFormat::Format_IMC2: + case QVideoSurfaceFormat::Format_IMC3: + case QVideoSurfaceFormat::Format_IMC4: + return QString(); + case QVideoSurfaceFormat::Format_AYUV444: + case QVideoSurfaceFormat::Format_AYUV444_Premultiplied: + return QStringLiteral(":/qtmultimedia/shaders/ayuv.frag.qsb"); + case QVideoSurfaceFormat::Format_ARGB32: + case QVideoSurfaceFormat::Format_ARGB32_Premultiplied: + case QVideoSurfaceFormat::Format_RGB32: + case QVideoSurfaceFormat::Format_BGRA32: + case QVideoSurfaceFormat::Format_BGRA32_Premultiplied: + case QVideoSurfaceFormat::Format_ABGR32: + case QVideoSurfaceFormat::Format_BGR32: + return QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb"); + case QVideoSurfaceFormat::Format_YUV420P: + case QVideoSurfaceFormat::Format_YUV422P: + case QVideoSurfaceFormat::Format_YV12: + return QStringLiteral(":/qtmultimedia/shaders/yuv_yv.frag.qsb"); + case QVideoSurfaceFormat::Format_UYVY: + return QStringLiteral(":/qtmultimedia/shaders/uyvy.frag.qsb"); + case QVideoSurfaceFormat::Format_YUYV: + return QStringLiteral(":/qtmultimedia/shaders/yuyv.frag.qsb"); + case QVideoSurfaceFormat::Format_NV12: + return QStringLiteral(":/qtmultimedia/shaders/nv12.frag.qsb"); + case QVideoSurfaceFormat::Format_NV21: + return QStringLiteral(":/qtmultimedia/shaders/nv21.frag.qsb"); + case QVideoSurfaceFormat::Format_P010LE: + case QVideoSurfaceFormat::Format_P016LE: + return QStringLiteral(":/qtmultimedia/shaders/p010le.frag.qsb"); + case QVideoSurfaceFormat::Format_P010BE: + case QVideoSurfaceFormat::Format_P016BE: + return QStringLiteral(":/qtmultimedia/shaders/p010be.frag.qsb"); + } +} + + +static QMatrix4x4 colorMatrix(QVideoSurfaceFormat::YCbCrColorSpace colorSpace) +{ + switch (colorSpace) { + case QVideoSurfaceFormat::YCbCr_JPEG: + return QMatrix4x4( + 1.0f, 0.000f, 1.402f, -0.701f, + 1.0f, -0.344f, -0.714f, 0.529f, + 1.0f, 1.772f, 0.000f, -0.886f, + 0.0f, 0.000f, 0.000f, 1.0000f); + case QVideoSurfaceFormat::YCbCr_BT709: + case QVideoSurfaceFormat::YCbCr_xvYCC709: + return QMatrix4x4( + 1.164f, 0.000f, 1.793f, -0.5727f, + 1.164f, -0.534f, -0.213f, 0.3007f, + 1.164f, 2.115f, 0.000f, -1.1302f, + 0.0f, 0.000f, 0.000f, 1.0000f); + default: //BT 601: + return QMatrix4x4( + 1.164f, 0.000f, 1.596f, -0.8708f, + 1.164f, -0.392f, -0.813f, 0.5296f, + 1.164f, 2.017f, 0.000f, -1.081f, + 0.0f, 0.000f, 0.000f, 1.0000f); + } +} + +QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &transform, float opacity) +{ + static constexpr float pw[3] = {}; + const float *planeWidth = pw; + + switch (format.pixelFormat()) { + case QVideoSurfaceFormat::Format_Invalid: + case QVideoSurfaceFormat::Format_Jpeg: + + case QVideoSurfaceFormat::Format_RGB24: + case QVideoSurfaceFormat::Format_RGB565: + case QVideoSurfaceFormat::Format_RGB555: + case QVideoSurfaceFormat::Format_ARGB8565_Premultiplied: + case QVideoSurfaceFormat::Format_BGR24: + case QVideoSurfaceFormat::Format_BGR565: + case QVideoSurfaceFormat::Format_BGR555: + case QVideoSurfaceFormat::Format_BGRA5658_Premultiplied: + + case QVideoSurfaceFormat::Format_Y8: + case QVideoSurfaceFormat::Format_Y16: + + case QVideoSurfaceFormat::Format_YUV444: + + case QVideoSurfaceFormat::Format_IMC1: + case QVideoSurfaceFormat::Format_IMC2: + case QVideoSurfaceFormat::Format_IMC3: + case QVideoSurfaceFormat::Format_IMC4: + return QByteArray(); + case QVideoSurfaceFormat::Format_ARGB32: + case QVideoSurfaceFormat::Format_ARGB32_Premultiplied: + case QVideoSurfaceFormat::Format_RGB32: + case QVideoSurfaceFormat::Format_BGRA32: + case QVideoSurfaceFormat::Format_BGRA32_Premultiplied: + case QVideoSurfaceFormat::Format_ABGR32: + case QVideoSurfaceFormat::Format_BGR32: { + // { matrix4x4, opacity } + QByteArray buf(16*4 + 4, Qt::Uninitialized); + char *data = buf.data(); + memcpy(data, transform.constData(), 64); + memcpy(data + 64, &opacity, 4); + return buf; + } + case QVideoSurfaceFormat::Format_AYUV444: + case QVideoSurfaceFormat::Format_AYUV444_Premultiplied: { + static constexpr float pw[] = { 1, 0, 0 }; + planeWidth = pw; + break; + } + case QVideoSurfaceFormat::Format_YUV420P: + case QVideoSurfaceFormat::Format_YUV422P: + case QVideoSurfaceFormat::Format_YV12: { + static constexpr float pw[] = { 1, 1, 1 }; + planeWidth = pw; + break; + } + case QVideoSurfaceFormat::Format_UYVY: + case QVideoSurfaceFormat::Format_YUYV: { + static constexpr float pw[] = { 1, 1, 0 }; + planeWidth = pw; + break; + } + case QVideoSurfaceFormat::Format_NV12: + case QVideoSurfaceFormat::Format_NV21: + case QVideoSurfaceFormat::Format_P010LE: + case QVideoSurfaceFormat::Format_P010BE: + case QVideoSurfaceFormat::Format_P016LE: + case QVideoSurfaceFormat::Format_P016BE: { + static constexpr float pw[] = { 1, 1, 0 }; + planeWidth = pw; + break; + } + } + // { matrix4x4, colorMatrix, opacity, planeWidth[3] } + QByteArray buf(64*2 + 4 + 3*4, Qt::Uninitialized); + char *data = buf.data(); + memcpy(data, transform.constData(), 64); + memcpy(data + 64, colorMatrix(format.yCbCrColorSpace()).constData(), 64); + memcpy(data + 64 + 64, &opacity, 4); + memcpy(data + 64 + 64 + 4, planeWidth, 3*4); + return buf; +} + +int updateRhiTextures(QVideoFrame frame, QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures) +{ + QVideoSurfaceFormat fmt = frame.surfaceFormat(); + QVideoSurfaceFormat::PixelFormat pixelFormat = fmt.pixelFormat(); + QSize size = fmt.frameSize(); + + const TextureDescription *description = descriptions + pixelFormat; + + QSize planeSizes[TextureDescription::maxPlanes]; + for (int plane = 0; plane < description->nplanes; ++plane) + planeSizes[plane] = QSize(size.width()/description->sizeScale[plane].x, size.height()/description->sizeScale[plane].y); + + if (frame.handleType() == QVideoFrame::RhiTextureHandle) { + for (int plane = 0; plane < description->nplanes; ++plane) { + quint64 nativeTexture = frame.textureHandle(plane); + Q_ASSERT(nativeTexture); + textures[plane] = rhi->newTexture(description->textureFormat[plane], planeSizes[plane], 1, {}); + textures[plane]->createFrom({nativeTexture, 0}); + } + return description->nplanes; + } + + // need to upload textures + bool mapped = frame.map(QVideoFrame::ReadOnly); + if (!mapped) { + qWarning() << "could not map data of QVideoFrame for upload"; + return 0; + } + + Q_ASSERT(frame.planeCount() == description->nplanes); + for (int plane = 0; plane < description->nplanes; ++plane) { + + bool needsRebuild = !textures[plane] || textures[plane]->pixelSize() != planeSizes[plane]; + if (!textures[plane]) + textures[plane] = rhi->newTexture(description->textureFormat[plane], planeSizes[plane], 1, {}); + + if (needsRebuild) { + textures[plane]->setPixelSize(planeSizes[plane]); + bool created = textures[plane]->create(); + if (!created) { + qWarning("Failed to create texture (size %dx%d)", planeSizes[plane].width(), planeSizes[plane].height()); + return 0; + } + } + + QRhiTextureSubresourceUploadDescription subresDesc(frame.bits(plane), frame.bytesPerLine(plane)*planeSizes[plane].height()); + subresDesc.setSourceSize(planeSizes[plane]); + subresDesc.setDestinationTopLeft(QPoint(0, 0)); + QRhiTextureUploadEntry entry(0, 0, subresDesc); + QRhiTextureUploadDescription desc({ entry }); + resourceUpdates->uploadTexture(textures[plane], desc); + } + return description->nplanes; +} + +} diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h b/src/multimedia/video/qvideotexturehelper_p.h index 61aa04873..4b369594a 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h +++ b/src/multimedia/video/qvideotexturehelper_p.h @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Toolkit. @@ -37,49 +37,41 @@ ** ****************************************************************************/ -#ifndef QSGVIDEONODE_TEXTURE_H -#define QSGVIDEONODE_TEXTURE_H +#ifndef QVIDEOTEXTUREHELPER_H +#define QVIDEOTEXTUREHELPER_H -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qsgvideonode_p.h> -#include <QtMultimedia/qvideosurfaceformat.h> +#include <qvideosurfaceformat.h> +#include <private/qrhi_p.h> QT_BEGIN_NAMESPACE -class QSGVideoMaterial_Texture; +class QVideoFrame; -class QSGVideoNode_Texture : public QSGVideoNode +namespace QVideoTextureHelper { -public: - QSGVideoNode_Texture(const QVideoSurfaceFormat &format); - ~QSGVideoNode_Texture(); - QVideoSurfaceFormat::PixelFormat pixelFormat() const override { - return m_format.pixelFormat(); - } - void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; +struct TextureDescription +{ + static constexpr int maxPlanes = 3; + struct SizeScale { + int x; + int y; + }; -private: - QVideoSurfaceFormat m_format; - QSGVideoMaterial_Texture *m_material; - QVideoFrame m_frame; + int nplanes; + QRhiTexture::Format textureFormat[maxPlanes]; + SizeScale sizeScale[maxPlanes]; }; -class QSGVideoNodeFactory_Texture : public QSGVideoNodeFactoryInterface { -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override; -}; +Q_MULTIMEDIA_EXPORT const TextureDescription *textureDescription(QVideoSurfaceFormat::PixelFormat format); + +Q_MULTIMEDIA_EXPORT QString vertexShaderFileName(QVideoSurfaceFormat::PixelFormat format); +Q_MULTIMEDIA_EXPORT QString fragmentShaderFileName(QVideoSurfaceFormat::PixelFormat format); +Q_MULTIMEDIA_EXPORT QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &transform, float opacity); +Q_MULTIMEDIA_EXPORT int updateRhiTextures(QVideoFrame frame, QRhi *rhi, + QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures); + +} QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/CMakeLists.txt b/src/qtmultimediaquicktools/CMakeLists.txt index 732fab664..96657b9ef 100644 --- a/src/qtmultimediaquicktools/CMakeLists.txt +++ b/src/qtmultimediaquicktools/CMakeLists.txt @@ -11,9 +11,6 @@ qt_add_module(MultimediaQuick qdeclarativevideooutput.cpp qdeclarativevideooutput_p.h qdeclarativevideooutput_render.cpp qdeclarativevideooutput_render_p.h qsgvideonode_p.cpp qsgvideonode_p.h - qsgvideonode_rgb.cpp qsgvideonode_rgb_p.h - qsgvideonode_texture.cpp qsgvideonode_texture_p.h - qsgvideonode_yuv.cpp qsgvideonode_yuv_p.h qsgvideotexture.cpp qsgvideotexture_p.h qtmultimediaquickdefs_p.h PUBLIC_LIBRARIES diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp index a45350fdc..f62a6c891 100644 --- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp +++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp @@ -55,33 +55,10 @@ QT_BEGIN_NAMESPACE Q_DECLARE_LOGGING_CATEGORY(qLcVideo) -Q_GLOBAL_STATIC_WITH_ARGS(QMediaPluginLoader, videoNodeFactoryLoader, - (QSGVideoNodeFactoryInterface_iid, QLatin1String("video/videonode"), Qt::CaseInsensitive)) - QDeclarativeVideoBackend::QDeclarativeVideoBackend(QDeclarativeVideoOutput *parent) : q(parent), m_frameChanged(false) { - // Prioritize the plugin requested by the environment - QString requestedVideoNode = QString::fromLatin1(qgetenv("QT_VIDEONODE")); - - const auto keys = videoNodeFactoryLoader()->keys(); - for (const QString &key : keys) { - QObject *instance = videoNodeFactoryLoader()->instance(key); - QSGVideoNodeFactoryInterface* plugin = qobject_cast<QSGVideoNodeFactoryInterface*>(instance); - if (plugin) { - if (key == requestedVideoNode) - m_videoNodeFactories.prepend(plugin); - else - m_videoNodeFactories.append(plugin); - qCDebug(qLcVideo) << "found videonode plugin" << key << plugin; - } - } - - // Append existing node factories as fallback if we have no plugins - m_videoNodeFactories.append(&m_i420Factory); - m_videoNodeFactories.append(&m_rgbFactory); - m_videoNodeFactories.append(&m_textureFactory); } QDeclarativeVideoBackend::~QDeclarativeVideoBackend() @@ -277,15 +254,8 @@ QSGNode *QDeclarativeVideoBackend::updatePaintNode(QSGNode *oldNode, // Get a node that supports our frame. The surface is irrelevant, our // QSGVideoItemSurface supports (logically) anything. updateGeometry(); - for (QSGVideoNodeFactoryInterface* factory : qAsConst(m_videoNodeFactories)) { - videoNode = factory->createNode(m_surfaceFormat); - if (videoNode) { - qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType() - << " Supported formats for the handle by this node:" - << factory->supportedPixelFormats(m_frame.handleType()); - break; - } - } + videoNode = new QSGVideoNode(m_surfaceFormat); + qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType(); } } diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h index 95b3f1ebd..4fa6e5980 100644 --- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h +++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h @@ -54,9 +54,7 @@ #include <QtQuick/qquickitem.h> #include <QtQuick/qsgnode.h> -#include <private/qsgvideonode_yuv_p.h> -#include <private/qsgvideonode_rgb_p.h> -#include <private/qsgvideonode_texture_p.h> +#include <private/qsgvideonode_p.h> #include <QtCore/qmutex.h> @@ -94,16 +92,12 @@ private: void scheduleDeleteFilterResources(); QDeclarativeVideoOutput *q; - QList<QSGVideoNodeFactoryInterface*> m_videoNodeFactories; mutable QVideoSink *m_sink = nullptr; QVideoSurfaceFormat m_surfaceFormat; QVideoFrame m_frame; QVideoFrame m_frameOnFlush; bool m_frameChanged; - QSGVideoNodeFactory_YUV m_i420Factory; - QSGVideoNodeFactory_RGB m_rgbFactory; - QSGVideoNodeFactory_Texture m_textureFactory; QMutex m_frameMutex; QRectF m_renderedRect; // Destination pixel coordinates, clipped QRectF m_sourceTextureRect; // Source texture coordinates diff --git a/src/qtmultimediaquicktools/qsgvideonode_p.cpp b/src/qtmultimediaquicktools/qsgvideonode_p.cpp index 9cb81902e..30faf95ef 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_p.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_p.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Toolkit. @@ -38,15 +38,13 @@ ****************************************************************************/ #include "qsgvideonode_p.h" +#include <QtQuick/qsgmaterial.h> +#include "qsgvideotexture_p.h" +#include <QtMultimedia/private/qvideotexturehelper_p.h> +#include <qmutex.h> QT_BEGIN_NAMESPACE -QSGVideoNode::QSGVideoNode() - : m_orientation(-1) -{ - setFlag(QSGNode::OwnsGeometry); -} - /* Helpers */ static inline void qSetGeom(QSGGeometry::TexturedPoint2D *v, const QPointF &p) { @@ -124,6 +122,161 @@ void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &tex markDirty(DirtyGeometry); } -QSGVideoNodeFactoryInterface::~QSGVideoNodeFactoryInterface() = default; +class QSGVideoMaterialRhiShader : public QSGMaterialShader +{ +public: + QSGVideoMaterialRhiShader(const QVideoSurfaceFormat &format) + : m_format(format) + { + setShaderFileName(VertexStage, m_format.vertexShaderFileName()); + setShaderFileName(FragmentStage, m_format.fragmentShaderFileName()); + } + + bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, + QSGMaterial *oldMaterial) override; + + void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, + QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; + +protected: + QVideoSurfaceFormat m_format; + float m_planeWidth[3] = {0, 0, 0}; + QMatrix4x4 m_colorMatrix; +}; + +class QSGVideoMaterial : public QSGMaterial +{ +public: + QSGVideoMaterial(const QVideoSurfaceFormat &format); + + [[nodiscard]] QSGMaterialType *type() const override { + static QSGMaterialType type[QVideoSurfaceFormat::NPixelFormats]; + return &type[m_format.pixelFormat()]; + } + + [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { + return new QSGVideoMaterialRhiShader(m_format); + } + + int compare(const QSGMaterial *other) const override { + const QSGVideoMaterial *m = static_cast<const QSGVideoMaterial *>(other); + + qint64 diff = m_textures[0]->comparisonKey() - m->m_textures[0]->comparisonKey(); + if (!diff) + diff = m_textures[1]->comparisonKey() - m->m_textures[1]->comparisonKey(); + if (!diff) + diff = m_textures[2]->comparisonKey() - m->m_textures[2]->comparisonKey(); + + return diff < 0 ? -1 : (diff > 0 ? 1 : 0); + } + + void updateBlending() { + // ### respect video formats with Alpha + setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); + } + + void setCurrentFrame(const QVideoFrame &frame) { + QMutexLocker lock(&m_frameMutex); + m_frame = frame; + m_texturesDirty = true; + } + + void updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates); + + QVideoSurfaceFormat m_format; + float m_planeWidth[3]; + float m_opacity; + + QMutex m_frameMutex; + bool m_texturesDirty = false; + QVideoFrame m_frame; + + QScopedPointer<QSGVideoTexture> m_textures[3]; +}; + +void QSGVideoMaterial::updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) +{ + QMutexLocker locker(&m_frameMutex); + if (!m_texturesDirty) + return; + + // update and upload all textures + QRhiTexture *textures[3] = {}; + for (int i = 0; i < 3; ++i) { + if (m_textures[i].data()) + textures[i] = m_textures[i].data()->rhiTexture(); + else + textures[i] = nullptr; + } + + QVideoTextureHelper::updateRhiTextures(m_frame, rhi, resourceUpdates, textures); + + for (int i = 0; i < 3; ++i) { + if (m_textures[i].data()) + m_textures[i].data()->setRhiTexture(textures[i]); + } +} + + +bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, + QSGMaterial *oldMaterial) +{ + Q_UNUSED(oldMaterial); + + auto m = static_cast<QSGVideoMaterial *>(newMaterial); + + if (!state.isMatrixDirty() && !state.isOpacityDirty()) + return false; + + if (state.isOpacityDirty()) { + m->m_opacity = state.opacity(); + m->updateBlending(); + } + + QByteArray *buf = state.uniformData(); + *buf = m_format.uniformData(state.combinedMatrix(), state.opacity()); + + return true; +} + +void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, + QSGMaterial *newMaterial, QSGMaterial *oldMaterial) +{ + Q_UNUSED(oldMaterial); + if (binding < 1 || binding > 3) + return; + + auto m = static_cast<QSGVideoMaterial *>(newMaterial); + + m->updateTextures(state.rhi(), state.resourceUpdateBatch()); + + *texture = m->m_textures[binding - 1].data(); +} + +QSGVideoMaterial::QSGVideoMaterial(const QVideoSurfaceFormat &format) : + m_format(format), + m_opacity(1.0) +{ + m_textures[0].reset(new QSGVideoTexture); + m_textures[1].reset(new QSGVideoTexture); + m_textures[2].reset(new QSGVideoTexture); + + setFlag(Blending, false); +} + +QSGVideoNode::QSGVideoNode(const QVideoSurfaceFormat &format) + : m_orientation(-1), + m_format(format) +{ + setFlag(QSGNode::OwnsMaterial); + m_material = new QSGVideoMaterial(format); + setMaterial(m_material); +} + +void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame, FrameFlags) +{ + m_material->setCurrentFrame(frame); + markDirty(DirtyMaterial); +} QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_p.h b/src/qtmultimediaquicktools/qsgvideonode_p.h index 7d71cb9f5..36239cdcd 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_p.h +++ b/src/qtmultimediaquicktools/qsgvideonode_p.h @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Toolkit. @@ -60,7 +60,9 @@ QT_BEGIN_NAMESPACE -class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNode : public QSGGeometryNode +class QSGVideoMaterial; + +class QSGVideoNode : public QSGGeometryNode { public: enum FrameFlag { @@ -68,10 +70,12 @@ public: }; Q_DECLARE_FLAGS(FrameFlags, FrameFlag) - QSGVideoNode(); + QSGVideoNode(const QVideoSurfaceFormat &format); - virtual void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) = 0; - virtual QVideoSurfaceFormat::PixelFormat pixelFormat() const = 0; + QVideoSurfaceFormat::PixelFormat pixelFormat() const { + return m_format.pixelFormat(); + } + void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags); void setTexturedRectGeometry(const QRectF &boundingRect, const QRectF &textureRect, int orientation); @@ -79,30 +83,12 @@ private: QRectF m_rect; QRectF m_textureRect; int m_orientation; -}; - -Q_DECLARE_OPERATORS_FOR_FLAGS(QSGVideoNode::FrameFlags) - -class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNodeFactoryInterface -{ -public: - virtual ~QSGVideoNodeFactoryInterface(); - virtual QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const = 0; - virtual QSGVideoNode *createNode(const QVideoSurfaceFormat &format) = 0; + QVideoSurfaceFormat m_format; + QSGVideoMaterial *m_material; }; -#define QSGVideoNodeFactoryInterface_iid "org.qt-project.qt.sgvideonodefactory/5.2" -Q_DECLARE_INTERFACE(QSGVideoNodeFactoryInterface, QSGVideoNodeFactoryInterface_iid) - -class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNodeFactoryPlugin : public QObject, public QSGVideoNodeFactoryInterface -{ - Q_OBJECT - Q_INTERFACES(QSGVideoNodeFactoryInterface) -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override = 0; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override = 0; -}; +Q_DECLARE_OPERATORS_FOR_FLAGS(QSGVideoNode::FrameFlags) QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp b/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp deleted file mode 100644 index 064cf43b0..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qsgvideonode_rgb_p.h" -#include "qsgvideotexture_p.h" -#include <QtQuick/qsgmaterial.h> -#include <QtCore/qmutex.h> - -QT_BEGIN_NAMESPACE - -QList<QVideoSurfaceFormat::PixelFormat> QSGVideoNodeFactory_RGB::supportedPixelFormats( - QVideoFrame::HandleType handleType) const -{ - QList<QVideoSurfaceFormat::PixelFormat> pixelFormats; - - if (handleType == QVideoFrame::NoHandle) { - pixelFormats.append(QVideoSurfaceFormat::Format_RGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_ARGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_ARGB32_Premultiplied); - pixelFormats.append(QVideoSurfaceFormat::Format_BGR32); - pixelFormats.append(QVideoSurfaceFormat::Format_BGRA32); - pixelFormats.append(QVideoSurfaceFormat::Format_RGB565); - } - - return pixelFormats; -} - -QSGVideoNode *QSGVideoNodeFactory_RGB::createNode(const QVideoSurfaceFormat &format) -{ - if (supportedPixelFormats(QVideoFrame::NoHandle).contains(format.pixelFormat())) - return new QSGVideoNode_RGB(format); - - return nullptr; -} - -class QSGVideoMaterialRhiShader_RGB : public QSGMaterialShader -{ -public: - QSGVideoMaterialRhiShader_RGB(const QVideoSurfaceFormat &format) - : m_format(format) - { - setShaderFileName(VertexStage, format.vertexShaderFileName()); - setShaderFileName(FragmentStage, format.fragmentShaderFileName()); - } - - bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) override; - - void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; - QVideoSurfaceFormat m_format; -}; - -class QSGVideoMaterial_RGB : public QSGMaterial -{ -public: - QSGVideoMaterial_RGB(const QVideoSurfaceFormat &format) : - m_format(format), - m_opacity(1.0) - { - setFlag(Blending, false); - m_texture.reset(new QSGVideoTexture); - } - - [[nodiscard]] QSGMaterialType *type() const override { - static QSGMaterialType normalType; - return &normalType; - } - - [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { - return new QSGVideoMaterialRhiShader_RGB(m_format); - } - - int compare(const QSGMaterial *other) const override { - const QSGVideoMaterial_RGB *m = static_cast<const QSGVideoMaterial_RGB *>(other); - - const qint64 diff = m_texture->comparisonKey() - m->m_texture->comparisonKey(); - return diff < 0 ? -1 : (diff > 0 ? 1 : 0); - } - - void updateBlending() { - setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); - } - - void setVideoFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; - } - - QVideoFrame m_frame; - QMutex m_frameMutex; - QSize m_textureSize; - QVideoSurfaceFormat m_format; - float m_opacity; - QScopedPointer<QSGVideoTexture> m_texture; -}; - -bool QSGVideoMaterialRhiShader_RGB::updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - if (!state.isMatrixDirty() && !state.isOpacityDirty()) - return false; - - if (state.isOpacityDirty()) { - auto m = static_cast<QSGVideoMaterial_RGB *>(newMaterial); - m->m_opacity = state.opacity(); - m->updateBlending(); - } - - QByteArray *buf = state.uniformData(); - *buf = m_format.uniformData(state.combinedMatrix(), state.opacity()); - - return true; -} - -void QSGVideoMaterialRhiShader_RGB::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - if (binding < 1) - return; - - auto m = static_cast<QSGVideoMaterial_RGB *>(newMaterial); - - m->m_frameMutex.lock(); - auto frame = m->m_frame; - - if (frame.pixelFormat() == QVideoSurfaceFormat::Format_RGB565) // Format_RGB565 requires GL_UNSIGNED_SHORT_5_6_5 - frame = frame.toImage().convertToFormat(QImage::Format_RGBA8888_Premultiplied); - - auto format = QRhiTexture::RGBA8; - if (frame.pixelFormat() == QVideoSurfaceFormat::Format_RGB32 - || frame.pixelFormat() == QVideoSurfaceFormat::Format_ARGB32 - || frame.pixelFormat() == QVideoSurfaceFormat::Format_ARGB32_Premultiplied) - { - format = QRhiTexture::BGRA8; - } - - if (frame.isValid() && frame.map(QVideoFrame::ReadOnly)) { - m->m_texture->setData(format, frame.size(), frame.bits(), frame.bytesPerLine() * frame.height()); - frame.unmap(); - } - m->m_frameMutex.unlock(); - - m->m_texture->commitTextureOperations(state.rhi(), state.resourceUpdateBatch()); - *texture = m->m_texture.data(); -} - -QSGVideoNode_RGB::QSGVideoNode_RGB(const QVideoSurfaceFormat &format) : - m_format(format) -{ - setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_RGB(format); - setMaterial(m_material); -} - -QSGVideoNode_RGB::~QSGVideoNode_RGB() = default; - -void QSGVideoNode_RGB::setCurrentFrame(const QVideoFrame &frame, FrameFlags) -{ - m_material->setVideoFrame(frame); - markDirty(DirtyMaterial); -} - -QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb_p.h b/src/qtmultimediaquicktools/qsgvideonode_rgb_p.h deleted file mode 100644 index b42509c1f..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_rgb_p.h +++ /dev/null @@ -1,86 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QSGVIDEONODE_RGB_H -#define QSGVIDEONODE_RGB_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qsgvideonode_p.h> -#include <QtMultimedia/qvideosurfaceformat.h> - -QT_BEGIN_NAMESPACE - -class QSGVideoMaterial_RGB; - -class QSGVideoNode_RGB : public QSGVideoNode -{ -public: - QSGVideoNode_RGB(const QVideoSurfaceFormat &format); - ~QSGVideoNode_RGB(); - - QVideoSurfaceFormat::PixelFormat pixelFormat() const override { - return m_format.pixelFormat(); - } - void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; - -private: - QVideoSurfaceFormat m_format; - QSGVideoMaterial_RGB *m_material; - QVideoFrame m_frame; -}; - -class QSGVideoNodeFactory_RGB : public QSGVideoNodeFactoryInterface { -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override; -}; - -QT_END_NAMESPACE - -#endif // QSGVIDEONODE_RGB_H diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp deleted file mode 100644 index 8f363cd46..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp +++ /dev/null @@ -1,212 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qsgvideonode_texture_p.h" -#include "qsgvideotexture_p.h" -#include <private/qsgrhisupport_p.h> -#include <QtQuick/qsgmaterial.h> -#include <QtCore/qmutex.h> -#include <QtMultimedia/private/qtmultimediaglobal_p.h> - -QT_BEGIN_NAMESPACE - -QList<QVideoSurfaceFormat::PixelFormat> QSGVideoNodeFactory_Texture::supportedPixelFormats( - QVideoFrame::HandleType) const -{ - QList<QVideoSurfaceFormat::PixelFormat> pixelFormats; - - pixelFormats.append(QVideoSurfaceFormat::Format_RGB565); - pixelFormats.append(QVideoSurfaceFormat::Format_RGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_ARGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_BGR32); - pixelFormats.append(QVideoSurfaceFormat::Format_BGRA32); -#if !QT_CONFIG(gpu_vivante) - pixelFormats.append(QVideoSurfaceFormat::Format_ABGR32); -#endif - - return pixelFormats; -} - -QSGVideoNode *QSGVideoNodeFactory_Texture::createNode(const QVideoSurfaceFormat &format) -{ - if (supportedPixelFormats(QVideoFrame::NoHandle).contains(format.pixelFormat())) - return new QSGVideoNode_Texture(format); - - return nullptr; -} - -class QSGVideoMaterialRhiShader_Texture : public QSGMaterialShader -{ -public: - QSGVideoMaterialRhiShader_Texture() - { - setShaderFileName(VertexStage, QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb")); - setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb")); - } - - bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) override; - - void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; -}; - -class QSGVideoMaterialRhiShader_Texture_swizzle : public QSGVideoMaterialRhiShader_Texture -{ -public: - QSGVideoMaterialRhiShader_Texture_swizzle() - { - setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimedia/shaders/bgra.frag.qsb")); - } -}; - -class QSGVideoMaterial_Texture : public QSGMaterial -{ -public: - QSGVideoMaterial_Texture(const QVideoSurfaceFormat &format) : - m_format(format), - m_opacity(1.0) - { - setFlag(Blending, false); - m_texture.reset(new QSGVideoTexture); - } - - ~QSGVideoMaterial_Texture() override = default; - - [[nodiscard]] QSGMaterialType *type() const override { - static QSGMaterialType normalType, swizzleType; - return needsSwizzling() ? &swizzleType : &normalType; - } - - [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { - return needsSwizzling() ? new QSGVideoMaterialRhiShader_Texture_swizzle - : new QSGVideoMaterialRhiShader_Texture; - } - - int compare(const QSGMaterial *other) const override { - const QSGVideoMaterial_Texture *m = static_cast<const QSGVideoMaterial_Texture *>(other); - - const qint64 diff = m_texture->comparisonKey() - m->m_texture->comparisonKey(); - return diff < 0 ? -1 : (diff > 0 ? 1 : 0); - } - - void updateBlending() { - setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); - } - - void setVideoFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; - } - - QVideoFrame m_frame; - QMutex m_frameMutex; - QVideoSurfaceFormat m_format; - quint64 m_textureId; - float m_opacity; - QScopedPointer<QSGVideoTexture> m_texture; - -private: - [[nodiscard]] bool needsSwizzling() const { - return m_format.pixelFormat() == QVideoSurfaceFormat::Format_RGB32 - || m_format.pixelFormat() == QVideoSurfaceFormat::Format_ARGB32; - } -}; - -bool QSGVideoMaterialRhiShader_Texture::updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - bool changed = false; - QByteArray *buf = state.uniformData(); - - if (state.isMatrixDirty()) { - memcpy(buf->data(), state.combinedMatrix().constData(), 64); - changed = true; - } - - if (state.isOpacityDirty()) { - auto m = static_cast<QSGVideoMaterial_Texture *>(newMaterial); - m->m_opacity = state.opacity(); - m->updateBlending(); - memcpy(buf->data() + 64, &m->m_opacity, 4); - changed = true; - } - - return changed; -} - -void QSGVideoMaterialRhiShader_Texture::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - if (binding < 1) - return; - - auto m = static_cast<QSGVideoMaterial_Texture *>(newMaterial); - - m->m_frameMutex.lock(); - auto size = m->m_frame.size(); - if (m->m_frame.isValid()) - m->m_textureId = m->m_frame.textureHandle(0); - m->m_frameMutex.unlock(); - - m->m_texture->setNativeObject(m->m_textureId, size); - m->m_texture->commitTextureOperations(state.rhi(), state.resourceUpdateBatch()); - *texture = m->m_texture.data(); -} - -QSGVideoNode_Texture::QSGVideoNode_Texture(const QVideoSurfaceFormat &format) : - m_format(format) -{ - setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_Texture(format); - setMaterial(m_material); -} - -QSGVideoNode_Texture::~QSGVideoNode_Texture() = default; - -void QSGVideoNode_Texture::setCurrentFrame(const QVideoFrame &frame, FrameFlags) -{ - m_material->setVideoFrame(frame); - markDirty(DirtyMaterial); -} - -QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp deleted file mode 100644 index d305e52a7..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp +++ /dev/null @@ -1,424 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qsgvideonode_yuv_p.h" -#include "qsgvideotexture_p.h" -#include <QtCore/qmutex.h> -#include <QtQuick/qsgmaterial.h> - -QT_BEGIN_NAMESPACE - -QList<QVideoSurfaceFormat::PixelFormat> QSGVideoNodeFactory_YUV::supportedPixelFormats( - QVideoFrame::HandleType) const -{ - QList<QVideoSurfaceFormat::PixelFormat> formats; - - formats << QVideoSurfaceFormat::Format_YUV420P << QVideoSurfaceFormat::Format_YV12 << QVideoSurfaceFormat::Format_YUV422P - << QVideoSurfaceFormat::Format_NV12 << QVideoSurfaceFormat::Format_NV21 - << QVideoSurfaceFormat::Format_UYVY << QVideoSurfaceFormat::Format_YUYV - << QVideoSurfaceFormat::Format_P010BE << QVideoSurfaceFormat::Format_P010LE - << QVideoSurfaceFormat::Format_P016BE << QVideoSurfaceFormat::Format_P016LE - ; - - return formats; -} - -QSGVideoNode *QSGVideoNodeFactory_YUV::createNode(const QVideoSurfaceFormat &format) -{ - if (supportedPixelFormats(QVideoFrame::NoHandle).contains(format.pixelFormat())) - return new QSGVideoNode_YUV(format); - - return nullptr; -} - -class QSGVideoMaterialRhiShader_YUV : public QSGMaterialShader -{ -public: - QSGVideoMaterialRhiShader_YUV(const QVideoSurfaceFormat &format) - : m_format(format) - { - setShaderFileName(VertexStage, m_format.vertexShaderFileName()); - setShaderFileName(FragmentStage, m_format.fragmentShaderFileName()); - } - - bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) override; - - void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; - - virtual void mapFrame(QSGVideoMaterial_YUV *) = 0; - -protected: - QVideoSurfaceFormat m_format; - float m_planeWidth[3] = {0, 0, 0}; - QMatrix4x4 m_colorMatrix; -}; - -class QSGVideoMaterialRhiShader_UYVY : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_UYVY(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterialRhiShader_YUV_YV : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_YUV_YV(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterialRhiShader_NV12 : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_NV12(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterialRhiShader_P010 : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_P010(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterial_YUV : public QSGMaterial -{ -public: - QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format); - - [[nodiscard]] QSGMaterialType *type() const override { - static QSGMaterialType biPlanarType, biPlanarSwizzleType, triPlanarType, uyvyType, yuyvType; - - switch (m_format.pixelFormat()) { - case QVideoSurfaceFormat::Format_NV12: - case QVideoSurfaceFormat::Format_P010BE: - case QVideoSurfaceFormat::Format_P010LE: - case QVideoSurfaceFormat::Format_P016LE: - case QVideoSurfaceFormat::Format_P016BE: - return &biPlanarType; - case QVideoSurfaceFormat::Format_NV21: - return &biPlanarSwizzleType; - case QVideoSurfaceFormat::Format_UYVY: - return &uyvyType; - case QVideoSurfaceFormat::Format_YUYV: - return &yuyvType; - default: // Currently: YUV420P, YUV422P and YV12 - return &triPlanarType; - } - } - - [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { - switch (m_format.pixelFormat()) { - case QVideoSurfaceFormat::Format_NV12: - case QVideoSurfaceFormat::Format_NV21: - return new QSGVideoMaterialRhiShader_NV12(m_format); - case QVideoSurfaceFormat::Format_UYVY: - case QVideoSurfaceFormat::Format_YUYV: - return new QSGVideoMaterialRhiShader_UYVY(m_format); - case QVideoSurfaceFormat::Format_P010LE: - case QVideoSurfaceFormat::Format_P016LE: - case QVideoSurfaceFormat::Format_P010BE: - case QVideoSurfaceFormat::Format_P016BE: - return new QSGVideoMaterialRhiShader_P010(m_format); - default: // Currently: YUV420P, YUV422P and YV12 - return new QSGVideoMaterialRhiShader_YUV_YV(m_format); - } - } - - int compare(const QSGMaterial *other) const override { - const QSGVideoMaterial_YUV *m = static_cast<const QSGVideoMaterial_YUV *>(other); - - qint64 diff = m_textures[0]->comparisonKey() - m->m_textures[0]->comparisonKey(); - if (!diff) - diff = m_textures[1]->comparisonKey() - m->m_textures[1]->comparisonKey(); - if (!diff) - diff = m_textures[2]->comparisonKey() - m->m_textures[2]->comparisonKey(); - - return diff < 0 ? -1 : (diff > 0 ? 1 : 0); - } - - void updateBlending() { - setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); - } - - void setCurrentFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; - } - - QVideoSurfaceFormat m_format; - float m_planeWidth[3]; - float m_opacity; - QMatrix4x4 m_colorMatrix; - QVideoFrame m_frame; - QMutex m_frameMutex; - QScopedPointer<QSGVideoTexture> m_textures[3]; -}; - -bool QSGVideoMaterialRhiShader_YUV::updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - auto m = static_cast<QSGVideoMaterial_YUV *>(newMaterial); - m->m_frameMutex.lock(); - mapFrame(m); - m->m_frameMutex.unlock(); - - if (!state.isMatrixDirty() && !state.isOpacityDirty() && m->m_colorMatrix == m_colorMatrix) - return false; - - if (state.isOpacityDirty()) { - m->m_opacity = state.opacity(); - m->updateBlending(); - } - - m_colorMatrix = m->m_colorMatrix; - - QByteArray *buf = state.uniformData(); - *buf = m_format.uniformData(state.combinedMatrix(), state.opacity()); - - return true; -} - -void QSGVideoMaterialRhiShader_YUV::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - if (binding < 1 || binding > 3) - return; - - auto m = static_cast<QSGVideoMaterial_YUV *>(newMaterial); - *texture = m->m_textures[binding - 1].data(); - (*texture)->commitTextureOperations(state.rhi(), state.resourceUpdateBatch()); -} - -void QSGVideoMaterialRhiShader_UYVY::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid() || !m->m_frame.map(QVideoFrame::ReadOnly)) - return; - - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - - m->m_planeWidth[0] = 1; - m->m_planeWidth[1] = 1; - - // Either r,b (YUYV) or g,a (UYVY) values are used as source of UV. - // Additionally U and V are set per 2 pixels hence only 1/2 of image width is used. - m->m_textures[0]->setData(QRhiTexture::RG8, m->m_frame.size(), - m->m_frame.bits(), m->m_frame.bytesPerLine() * fh); - m->m_textures[1]->setData(QRhiTexture::RGBA8, QSize(fw / 2, fh), - m->m_frame.bits(), m->m_frame.bytesPerLine() * fh); - - m->m_frame.unmap(); -} - -void QSGVideoMaterialRhiShader_YUV_YV::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid() || !m->m_frame.map(QVideoFrame::ReadOnly)) - return; - - int y = 0; - int u = m->m_frame.pixelFormat() == QVideoSurfaceFormat::Format_YV12 ? 2 : 1; - int v = m->m_frame.pixelFormat() == QVideoSurfaceFormat::Format_YV12 ? 1 : 2; - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - int uvHeight = m->m_frame.pixelFormat() == QVideoSurfaceFormat::Format_YUV422P ? fh : fh / 2; - - m->m_planeWidth[0] = float(fw) / m->m_frame.bytesPerLine(y); - m->m_planeWidth[1] = m->m_planeWidth[2] = float(fw) / (2 * m->m_frame.bytesPerLine(u)); - - m->m_textures[0]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(y), fh), - m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh); - m->m_textures[1]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(u), uvHeight), - m->m_frame.bits(u), m->m_frame.bytesPerLine(u) * uvHeight); - m->m_textures[2]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(v), uvHeight), - m->m_frame.bits(v), m->m_frame.bytesPerLine(v) * uvHeight); - - m->m_frame.unmap(); -} - -void QSGVideoMaterialRhiShader_NV12::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid()) - return; - - if (m->m_frame.handleType() == QVideoFrame::RhiTextureHandle) { - m->m_planeWidth[0] = m->m_planeWidth[1] = 1; - quint64 textures[2] = { m->m_frame.textureHandle(0), m->m_frame.textureHandle(1) }; - if (textures[0] && textures[1]) { - auto w = m->m_frame.size().width(); - auto h = m->m_frame.size().height(); - m->m_textures[0]->setNativeObject(textures[0], {w, h}, QRhiTexture::R8); - m->m_textures[1]->setNativeObject(textures[1], {w / 2, h / 2}, QRhiTexture::RG8); - } else { - qWarning() << "NV12/NV21 requires 2 textures"; - } - - return; - } - - if (!m->m_frame.map(QVideoFrame::ReadOnly)) { - qWarning()<< "NV12: Couldn't map frame"; - m->m_textures[0]->setData(QRhiTexture::RG8, QSize(1, 1), (const uchar *)"\0\0", 2); - m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(1, 1), (const uchar *)"\0\0\0\0", 4); - return; - } - - int y = 0; - int uv = 1; - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - - m->m_planeWidth[0] = m->m_planeWidth[1] = qreal(fw) / m->m_frame.bytesPerLine(y); - - m->m_textures[0]->setData(QRhiTexture::R8, m->m_frame.size(), - m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh); - m->m_textures[1]->setData(QRhiTexture::RG8, QSize(m->m_frame.bytesPerLine(uv) / 2 , fh / 2), - m->m_frame.bits(uv), m->m_frame.bytesPerLine(uv) * fh / 2); - - m->m_frame.unmap(); -} - -void QSGVideoMaterialRhiShader_P010::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid()) - return; - - if (m->m_frame.handleType() == QVideoFrame::RhiTextureHandle) { - m->m_planeWidth[0] = m->m_planeWidth[1] = 1; - quint64 textures[2] = { m->m_frame.textureHandle(0), m->m_frame.textureHandle(1) }; - if (textures[0] && textures[1]) { - auto w = m->m_frame.size().width(); - auto h = m->m_frame.size().height(); - m->m_textures[0]->setNativeObject(textures[0], {w, h}, QRhiTexture::RG8); - m->m_textures[1]->setNativeObject(textures[1], {w / 2, h / 2}, QRhiTexture::BGRA8); - } else { - qWarning() << "P010/P016 requires 2 textures"; - } - - return; - } - - if (!m->m_frame.map(QVideoFrame::ReadOnly)) { - qWarning()<< "NV12: Couldn't map frame"; - m->m_textures[0]->setData(QRhiTexture::RG8, QSize(1, 1), (const uchar *)"\0\0", 2); - m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(1, 1), (const uchar *)"\0\0\0\0", 4); - return; - } - - int y = 0; - int uv = 1; - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - - m->m_planeWidth[0] = m->m_planeWidth[1] = qreal(fw) / m->m_frame.bytesPerLine(y); - - // Use RG8 and BGRA8 textures for Y and UV. This allows us to pick up the 8 most significant - // bits of the values easily. Since our target surface is not HDR capable this will do for now. - m->m_textures[0]->setData(QRhiTexture::RG8, m->m_frame.size(), - m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh); - m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(m->m_frame.bytesPerLine(uv) / 4 , fh / 2), - m->m_frame.bits(uv), m->m_frame.bytesPerLine(uv) * fh / 2); - - m->m_frame.unmap(); -} - -QSGVideoMaterial_YUV::QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format) : - m_format(format), - m_opacity(1.0) -{ - m_textures[0].reset(new QSGVideoTexture); - m_textures[1].reset(new QSGVideoTexture); - m_textures[2].reset(new QSGVideoTexture); - - switch (format.yCbCrColorSpace()) { - case QVideoSurfaceFormat::YCbCr_JPEG: - m_colorMatrix = QMatrix4x4( - 1.0f, 0.000f, 1.402f, -0.701f, - 1.0f, -0.344f, -0.714f, 0.529f, - 1.0f, 1.772f, 0.000f, -0.886f, - 0.0f, 0.000f, 0.000f, 1.0000f); - break; - case QVideoSurfaceFormat::YCbCr_BT709: - case QVideoSurfaceFormat::YCbCr_xvYCC709: - m_colorMatrix = QMatrix4x4( - 1.164f, 0.000f, 1.793f, -0.5727f, - 1.164f, -0.534f, -0.213f, 0.3007f, - 1.164f, 2.115f, 0.000f, -1.1302f, - 0.0f, 0.000f, 0.000f, 1.0000f); - break; - default: //BT 601: - m_colorMatrix = QMatrix4x4( - 1.164f, 0.000f, 1.596f, -0.8708f, - 1.164f, -0.392f, -0.813f, 0.5296f, - 1.164f, 2.017f, 0.000f, -1.081f, - 0.0f, 0.000f, 0.000f, 1.0000f); - } - - setFlag(Blending, false); -} - -QSGVideoNode_YUV::QSGVideoNode_YUV(const QVideoSurfaceFormat &format) : - m_format(format) -{ - setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_YUV(format); - setMaterial(m_material); -} - -QSGVideoNode_YUV::~QSGVideoNode_YUV() = default; - -void QSGVideoNode_YUV::setCurrentFrame(const QVideoFrame &frame, FrameFlags) -{ - m_material->setCurrentFrame(frame); - markDirty(DirtyMaterial); -} - -QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv_p.h b/src/qtmultimediaquicktools/qsgvideonode_yuv_p.h deleted file mode 100644 index 55b3bc764..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_yuv_p.h +++ /dev/null @@ -1,86 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QSGVIDEONODE_YUV_H -#define QSGVIDEONODE_YUV_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qsgvideonode_p.h> -#include <QtMultimedia/qvideosurfaceformat.h> - -QT_BEGIN_NAMESPACE - -class QSGVideoMaterial_YUV; -class QSGVideoNode_YUV : public QSGVideoNode -{ -public: - QSGVideoNode_YUV(const QVideoSurfaceFormat &format); - ~QSGVideoNode_YUV(); - - QVideoSurfaceFormat::PixelFormat pixelFormat() const override { - return m_format.pixelFormat(); - } - void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; - -private: - void bindTexture(int id, int unit, int w, int h, const uchar *bits); - - QVideoSurfaceFormat m_format; - QSGVideoMaterial_YUV *m_material; -}; - -class QSGVideoNodeFactory_YUV : public QSGVideoNodeFactoryInterface { -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override; -}; - -QT_END_NAMESPACE - -#endif // QSGVIDEONODE_YUV_H diff --git a/src/qtmultimediaquicktools/qsgvideotexture.cpp b/src/qtmultimediaquicktools/qsgvideotexture.cpp index 18c060de7..40a8f71e1 100644 --- a/src/qtmultimediaquicktools/qsgvideotexture.cpp +++ b/src/qtmultimediaquicktools/qsgvideotexture.cpp @@ -161,4 +161,9 @@ void QSGVideoTexture::commitTextureOperations(QRhi *rhi, QRhiResourceUpdateBatch d_func()->updateRhiTexture(rhi, resourceUpdates); } +void QSGVideoTexture::setRhiTexture(QRhiTexture *texture) +{ + d_func()->m_texture.reset(texture); +} + QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideotexture_p.h b/src/qtmultimediaquicktools/qsgvideotexture_p.h index 221b891e3..869cae4ba 100644 --- a/src/qtmultimediaquicktools/qsgvideotexture_p.h +++ b/src/qtmultimediaquicktools/qsgvideotexture_p.h @@ -72,6 +72,7 @@ public: bool hasAlphaChannel() const override; bool hasMipmaps() const override; void commitTextureOperations(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) override; + void setRhiTexture(QRhiTexture *texture); void setData(QRhiTexture::Format f, const QSize &s, const uchar *data, int bytes); void setNativeObject(quint64 obj, const QSize &s, QRhiTexture::Format f = QRhiTexture::RGBA8); |