diff options
| author | Lars Knoll <lars.knoll@qt.io> | 2021-03-26 10:24:57 +0100 |
|---|---|---|
| committer | Lars Knoll <lars.knoll@qt.io> | 2021-04-07 14:51:23 +0000 |
| commit | 0022bca8f430d7eea7a5f7e39a1cb3719118af2a (patch) | |
| tree | ec32a61cfa5a1bf69404f124a786633a4c3a4d59 /src | |
| parent | f66c18bc51989ae5388b333c2b2804dd7b6e3f1a (diff) | |
Move the texture uploading code into Qt Multimedia
Qt Multimedia has now some generic helper functions to upload
the video data into textures. This way, we can greatly simplify
the rendering code in qtmultimediaquicktools.
Change-Id: I5b0e3eb96cbcf79fe5e9757697f11b0183132b17
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
Diffstat (limited to 'src')
18 files changed, 734 insertions, 1363 deletions
diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt index ce8f44f61..8a1010188 100644 --- a/src/multimedia/CMakeLists.txt +++ b/src/multimedia/CMakeLists.txt @@ -65,6 +65,7 @@ qt_internal_add_module(Multimedia video/qmemoryvideobuffer.cpp video/qmemoryvideobuffer_p.h video/qvideoframe.cpp video/qvideoframe.h video/qvideosink.cpp video/qvideosink.h + video/qvideotexturehelper.cpp video/qvideotexturehelper_p.h video/qvideoframeconversionhelper.cpp video/qvideoframeconversionhelper_p.h video/qvideooutputorientationhandler.cpp video/qvideooutputorientationhandler_p.h video/qvideosurfaceformat.cpp video/qvideosurfaceformat.h diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp index 2fa5f9f88..a4a985a5f 100644 --- a/src/multimedia/video/qvideoframe.cpp +++ b/src/multimedia/video/qvideoframe.cpp @@ -49,6 +49,7 @@ #include <qpair.h> #include <qsize.h> #include <qvariant.h> +#include <private/qrhi_p.h> #include <QDebug> @@ -812,7 +813,7 @@ int QVideoFrame::planeCount() const \internal Returns a texture id to the video frame's buffers. */ -quint64 QVideoFrame::textureHandle(int plane) +quint64 QVideoFrame::textureHandle(int plane) const { return d->buffer->textureHandle(plane); } diff --git a/src/multimedia/video/qvideoframe.h b/src/multimedia/video/qvideoframe.h index ea01ccb92..d2c371c3f 100644 --- a/src/multimedia/video/qvideoframe.h +++ b/src/multimedia/video/qvideoframe.h @@ -53,6 +53,9 @@ QT_BEGIN_NAMESPACE class QSize; class QVideoFramePrivate; class QAbstractVideoBuffer; +class QRhi; +class QRhiResourceUpdateBatch; +class QRhiTexture; class Q_MULTIMEDIA_EXPORT QVideoFrame { @@ -113,7 +116,7 @@ public: int mappedBytes() const; int planeCount() const; - quint64 textureHandle(int plane); + quint64 textureHandle(int plane) const; qint64 startTime() const; void setStartTime(qint64 time); diff --git a/src/multimedia/video/qvideosurfaceformat.cpp b/src/multimedia/video/qvideosurfaceformat.cpp index 5271a12db..745fe8b32 100644 --- a/src/multimedia/video/qvideosurfaceformat.cpp +++ b/src/multimedia/video/qvideosurfaceformat.cpp @@ -38,6 +38,7 @@ ****************************************************************************/ #include "qvideosurfaceformat.h" +#include "qvideotexturehelper_p.h" #include <qdebug.h> #include <qlist.h> @@ -259,49 +260,7 @@ int QVideoSurfaceFormat::frameHeight() const int QVideoSurfaceFormat::nPlanes() const { - switch (d->pixelFormat) { - case Format_Invalid: - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - case Format_UYVY: - case Format_YUYV: - case Format_Y8: - case Format_Y16: - case Format_Jpeg: - return 1; - case Format_NV12: - case Format_NV21: - case Format_IMC2: - case Format_IMC4: - case Format_P010LE: - case Format_P010BE: - case Format_P016LE: - case Format_P016BE: - return 2; - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: - case Format_IMC1: - case Format_IMC3: - return 3; - } - return 0; + return QVideoTextureHelper::textureDescription(d->pixelFormat)->nplanes; } /*! @@ -438,211 +397,17 @@ QSize QVideoSurfaceFormat::sizeHint() const QString QVideoSurfaceFormat::vertexShaderFileName() const { - switch (d->pixelFormat) { - case Format_Invalid: - case Format_Jpeg: - - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - - case Format_Y8: - case Format_Y16: - - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - - case Format_IMC1: - case Format_IMC2: - case Format_IMC3: - case Format_IMC4: - return QString(); - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: - return QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb"); - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: - case Format_UYVY: - case Format_YUYV: - case Format_NV12: - case Format_NV21: - case Format_P010LE: - case Format_P010BE: - case Format_P016LE: - case Format_P016BE: - return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb"); - } + return QVideoTextureHelper::vertexShaderFileName(d->pixelFormat); } QString QVideoSurfaceFormat::fragmentShaderFileName() const { - switch (d->pixelFormat) { - case Format_Invalid: - case Format_Jpeg: - - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - - case Format_Y8: - case Format_Y16: - - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - - case Format_IMC1: - case Format_IMC2: - case Format_IMC3: - case Format_IMC4: - return QString(); - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: - return QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb"); - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: - return QStringLiteral(":/qtmultimedia/shaders/yuv_yv.frag.qsb"); - case Format_UYVY: - return QStringLiteral(":/qtmultimedia/shaders/uyvy.frag.qsb"); - case Format_YUYV: - return QStringLiteral(":/qtmultimedia/shaders/yuyv.frag.qsb"); - case Format_NV12: - return QStringLiteral(":/qtmultimedia/shaders/nv12.frag.qsb"); - case Format_NV21: - return QStringLiteral(":/qtmultimedia/shaders/nv21.frag.qsb"); - case Format_P010LE: - case Format_P016LE: - return QStringLiteral(":/qtmultimedia/shaders/p010le.frag.qsb"); - case Format_P010BE: - case Format_P016BE: - return QStringLiteral(":/qtmultimedia/shaders/p010be.frag.qsb"); - } -} - -static QMatrix4x4 colorMatrix(QVideoSurfaceFormat::YCbCrColorSpace colorSpace) -{ - switch (colorSpace) { - case QVideoSurfaceFormat::YCbCr_JPEG: - return QMatrix4x4( - 1.0f, 0.000f, 1.402f, -0.701f, - 1.0f, -0.344f, -0.714f, 0.529f, - 1.0f, 1.772f, 0.000f, -0.886f, - 0.0f, 0.000f, 0.000f, 1.0000f); - case QVideoSurfaceFormat::YCbCr_BT709: - case QVideoSurfaceFormat::YCbCr_xvYCC709: - return QMatrix4x4( - 1.164f, 0.000f, 1.793f, -0.5727f, - 1.164f, -0.534f, -0.213f, 0.3007f, - 1.164f, 2.115f, 0.000f, -1.1302f, - 0.0f, 0.000f, 0.000f, 1.0000f); - default: //BT 601: - return QMatrix4x4( - 1.164f, 0.000f, 1.596f, -0.8708f, - 1.164f, -0.392f, -0.813f, 0.5296f, - 1.164f, 2.017f, 0.000f, -1.081f, - 0.0f, 0.000f, 0.000f, 1.0000f); - } + return QVideoTextureHelper::fragmentShaderFileName(d->pixelFormat); } QByteArray QVideoSurfaceFormat::uniformData(const QMatrix4x4 &transform, float opacity) const { - static constexpr float pw[3] = {}; - const float *planeWidth = pw; - - switch (d->pixelFormat) { - case Format_Invalid: - case Format_Jpeg: - - case Format_RGB24: - case Format_RGB565: - case Format_RGB555: - case Format_ARGB8565_Premultiplied: - case Format_BGR24: - case Format_BGR565: - case Format_BGR555: - case Format_BGRA5658_Premultiplied: - - case Format_Y8: - case Format_Y16: - - case Format_AYUV444: - case Format_AYUV444_Premultiplied: - case Format_YUV444: - - case Format_IMC1: - case Format_IMC2: - case Format_IMC3: - case Format_IMC4: - return QByteArray(); - case Format_ARGB32: - case Format_ARGB32_Premultiplied: - case Format_RGB32: - case Format_BGRA32: - case Format_BGRA32_Premultiplied: - case Format_ABGR32: - case Format_BGR32: { - // { matrix4x4, opacity } - QByteArray buf(16*4 + 4, Qt::Uninitialized); - char *data = buf.data(); - memcpy(data, transform.constData(), 64); - memcpy(data + 64, &opacity, 4); - return buf; - } - case Format_YUV420P: - case Format_YUV422P: - case Format_YV12: { - static constexpr float pw[] = { 1, 1, 1 }; - planeWidth = pw; - break; - } - case Format_UYVY: - case Format_YUYV: { - static constexpr float pw[] = { 1, 1, 0 }; - planeWidth = pw; - break; - } - case Format_NV12: - case Format_NV21: - case Format_P010LE: - case Format_P010BE: - case Format_P016LE: - case Format_P016BE: { - static constexpr float pw[] = { 1, 1, 0 }; - planeWidth = pw; - break; - } - } - // { matrix4x4, colorMatrix, opacity, planeWidth[3] } - QByteArray buf(64*2 + 4 + 3*4, Qt::Uninitialized); - char *data = buf.data(); - memcpy(data, transform.constData(), 64); - memcpy(data + 64, colorMatrix(d->ycbcrColorSpace).constData(), 64); - memcpy(data + 64 + 64, &opacity, 4); - memcpy(data + 64 + 64 + 4, planeWidth, 3*4); - return buf; + return QVideoTextureHelper::uniformData(*this, transform, opacity); } diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp new file mode 100644 index 000000000..930456d7d --- /dev/null +++ b/src/multimedia/video/qvideotexturehelper.cpp @@ -0,0 +1,514 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qvideotexturehelper_p.h" +#include "qvideoframe.h" + +namespace QVideoTextureHelper +{ + +static const TextureDescription descriptions[QVideoSurfaceFormat::NPixelFormats] = { + // Format_Invalid + { 0, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat}, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ARGB32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ARGB32_Premultiplied + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB24 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB565 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_RGB555 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ARGB8565_Premultiplied + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGRA32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGRA32_Premultiplied + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_ABGR32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR32 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR24 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR565 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGR555 + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_BGRA5658_Premultiplied + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + + // Format_AYUV444 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_AYUV444_Premultiplied + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_YUV444 + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_YUV420P + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 2, 2 }, { 2, 2 } } + }, + // Format_YUV422P + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 2, 1 }, { 2, 1 } } + }, + // Format_YV12 + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 2, 2 }, { 2, 2 } } + }, + // Format_UYVY + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_YUYV + { 1, + { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_NV12 + { 2, + { QRhiTexture::RG8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_NV21 + { 2, + { QRhiTexture::RG8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_IMC1 + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_IMC2 + { 2, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_IMC3 + { 3, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_IMC4 + { 2, + { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_Y8 + { 1, + { QRhiTexture::R8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + // Format_Y16 + { 1, + { QRhiTexture::R16, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + }, + + // Format_P010LE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_P010BE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_P016LE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + // Format_P016BE + { 2, + { QRhiTexture::RG8, QRhiTexture::BGRA8, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 2, 2 }, { 1, 1 } } + }, + + // Format_Jpeg + { 1, + { QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat }, + { { 1, 1 }, { 1, 1 }, { 1, 1 } } + } +}; + + +const TextureDescription *textureDescription(QVideoSurfaceFormat::PixelFormat format) +{ + return descriptions + format; +} + +QString vertexShaderFileName(QVideoSurfaceFormat::PixelFormat format) +{ + switch (format) { + case QVideoSurfaceFormat::Format_Invalid: + case QVideoSurfaceFormat::Format_Jpeg: + + case QVideoSurfaceFormat::Format_RGB24: + case QVideoSurfaceFormat::Format_RGB565: + case QVideoSurfaceFormat::Format_RGB555: + case QVideoSurfaceFormat::Format_ARGB8565_Premultiplied: + case QVideoSurfaceFormat::Format_BGR24: + case QVideoSurfaceFormat::Format_BGR565: + case QVideoSurfaceFormat::Format_BGR555: + case QVideoSurfaceFormat::Format_BGRA5658_Premultiplied: + + case QVideoSurfaceFormat::Format_Y8: + case QVideoSurfaceFormat::Format_Y16: + + case QVideoSurfaceFormat::Format_YUV444: + + case QVideoSurfaceFormat::Format_IMC1: + case QVideoSurfaceFormat::Format_IMC2: + case QVideoSurfaceFormat::Format_IMC3: + case QVideoSurfaceFormat::Format_IMC4: + return QString(); + case QVideoSurfaceFormat::Format_AYUV444: + case QVideoSurfaceFormat::Format_AYUV444_Premultiplied: + return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb"); + case QVideoSurfaceFormat::Format_ARGB32: + case QVideoSurfaceFormat::Format_ARGB32_Premultiplied: + case QVideoSurfaceFormat::Format_RGB32: + case QVideoSurfaceFormat::Format_BGRA32: + case QVideoSurfaceFormat::Format_BGRA32_Premultiplied: + case QVideoSurfaceFormat::Format_ABGR32: + case QVideoSurfaceFormat::Format_BGR32: + return QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb"); + case QVideoSurfaceFormat::Format_YUV420P: + case QVideoSurfaceFormat::Format_YUV422P: + case QVideoSurfaceFormat::Format_YV12: + case QVideoSurfaceFormat::Format_UYVY: + case QVideoSurfaceFormat::Format_YUYV: + case QVideoSurfaceFormat::Format_NV12: + case QVideoSurfaceFormat::Format_NV21: + case QVideoSurfaceFormat::Format_P010LE: + case QVideoSurfaceFormat::Format_P010BE: + case QVideoSurfaceFormat::Format_P016LE: + case QVideoSurfaceFormat::Format_P016BE: + return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb"); + } +} + +QString fragmentShaderFileName(QVideoSurfaceFormat::PixelFormat format) +{ + switch (format) { + case QVideoSurfaceFormat::Format_Invalid: + case QVideoSurfaceFormat::Format_Jpeg: + + case QVideoSurfaceFormat::Format_RGB24: + case QVideoSurfaceFormat::Format_RGB565: + case QVideoSurfaceFormat::Format_RGB555: + case QVideoSurfaceFormat::Format_ARGB8565_Premultiplied: + case QVideoSurfaceFormat::Format_BGR24: + case QVideoSurfaceFormat::Format_BGR565: + case QVideoSurfaceFormat::Format_BGR555: + case QVideoSurfaceFormat::Format_BGRA5658_Premultiplied: + + case QVideoSurfaceFormat::Format_Y8: + case QVideoSurfaceFormat::Format_Y16: + + case QVideoSurfaceFormat::Format_YUV444: + + case QVideoSurfaceFormat::Format_IMC1: + case QVideoSurfaceFormat::Format_IMC2: + case QVideoSurfaceFormat::Format_IMC3: + case QVideoSurfaceFormat::Format_IMC4: + return QString(); + case QVideoSurfaceFormat::Format_AYUV444: + case QVideoSurfaceFormat::Format_AYUV444_Premultiplied: + return QStringLiteral(":/qtmultimedia/shaders/ayuv.frag.qsb"); + case QVideoSurfaceFormat::Format_ARGB32: + case QVideoSurfaceFormat::Format_ARGB32_Premultiplied: + case QVideoSurfaceFormat::Format_RGB32: + case QVideoSurfaceFormat::Format_BGRA32: + case QVideoSurfaceFormat::Format_BGRA32_Premultiplied: + case QVideoSurfaceFormat::Format_ABGR32: + case QVideoSurfaceFormat::Format_BGR32: + return QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb"); + case QVideoSurfaceFormat::Format_YUV420P: + case QVideoSurfaceFormat::Format_YUV422P: + case QVideoSurfaceFormat::Format_YV12: + return QStringLiteral(":/qtmultimedia/shaders/yuv_yv.frag.qsb"); + case QVideoSurfaceFormat::Format_UYVY: + return QStringLiteral(":/qtmultimedia/shaders/uyvy.frag.qsb"); + case QVideoSurfaceFormat::Format_YUYV: + return QStringLiteral(":/qtmultimedia/shaders/yuyv.frag.qsb"); + case QVideoSurfaceFormat::Format_NV12: + return QStringLiteral(":/qtmultimedia/shaders/nv12.frag.qsb"); + case QVideoSurfaceFormat::Format_NV21: + return QStringLiteral(":/qtmultimedia/shaders/nv21.frag.qsb"); + case QVideoSurfaceFormat::Format_P010LE: + case QVideoSurfaceFormat::Format_P016LE: + return QStringLiteral(":/qtmultimedia/shaders/p010le.frag.qsb"); + case QVideoSurfaceFormat::Format_P010BE: + case QVideoSurfaceFormat::Format_P016BE: + return QStringLiteral(":/qtmultimedia/shaders/p010be.frag.qsb"); + } +} + + +static QMatrix4x4 colorMatrix(QVideoSurfaceFormat::YCbCrColorSpace colorSpace) +{ + switch (colorSpace) { + case QVideoSurfaceFormat::YCbCr_JPEG: + return QMatrix4x4( + 1.0f, 0.000f, 1.402f, -0.701f, + 1.0f, -0.344f, -0.714f, 0.529f, + 1.0f, 1.772f, 0.000f, -0.886f, + 0.0f, 0.000f, 0.000f, 1.0000f); + case QVideoSurfaceFormat::YCbCr_BT709: + case QVideoSurfaceFormat::YCbCr_xvYCC709: + return QMatrix4x4( + 1.164f, 0.000f, 1.793f, -0.5727f, + 1.164f, -0.534f, -0.213f, 0.3007f, + 1.164f, 2.115f, 0.000f, -1.1302f, + 0.0f, 0.000f, 0.000f, 1.0000f); + default: //BT 601: + return QMatrix4x4( + 1.164f, 0.000f, 1.596f, -0.8708f, + 1.164f, -0.392f, -0.813f, 0.5296f, + 1.164f, 2.017f, 0.000f, -1.081f, + 0.0f, 0.000f, 0.000f, 1.0000f); + } +} + +QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &transform, float opacity) +{ + static constexpr float pw[3] = {}; + const float *planeWidth = pw; + + switch (format.pixelFormat()) { + case QVideoSurfaceFormat::Format_Invalid: + case QVideoSurfaceFormat::Format_Jpeg: + + case QVideoSurfaceFormat::Format_RGB24: + case QVideoSurfaceFormat::Format_RGB565: + case QVideoSurfaceFormat::Format_RGB555: + case QVideoSurfaceFormat::Format_ARGB8565_Premultiplied: + case QVideoSurfaceFormat::Format_BGR24: + case QVideoSurfaceFormat::Format_BGR565: + case QVideoSurfaceFormat::Format_BGR555: + case QVideoSurfaceFormat::Format_BGRA5658_Premultiplied: + + case QVideoSurfaceFormat::Format_Y8: + case QVideoSurfaceFormat::Format_Y16: + + case QVideoSurfaceFormat::Format_YUV444: + + case QVideoSurfaceFormat::Format_IMC1: + case QVideoSurfaceFormat::Format_IMC2: + case QVideoSurfaceFormat::Format_IMC3: + case QVideoSurfaceFormat::Format_IMC4: + return QByteArray(); + case QVideoSurfaceFormat::Format_ARGB32: + case QVideoSurfaceFormat::Format_ARGB32_Premultiplied: + case QVideoSurfaceFormat::Format_RGB32: + case QVideoSurfaceFormat::Format_BGRA32: + case QVideoSurfaceFormat::Format_BGRA32_Premultiplied: + case QVideoSurfaceFormat::Format_ABGR32: + case QVideoSurfaceFormat::Format_BGR32: { + // { matrix4x4, opacity } + QByteArray buf(16*4 + 4, Qt::Uninitialized); + char *data = buf.data(); + memcpy(data, transform.constData(), 64); + memcpy(data + 64, &opacity, 4); + return buf; + } + case QVideoSurfaceFormat::Format_AYUV444: + case QVideoSurfaceFormat::Format_AYUV444_Premultiplied: { + static constexpr float pw[] = { 1, 0, 0 }; + planeWidth = pw; + break; + } + case QVideoSurfaceFormat::Format_YUV420P: + case QVideoSurfaceFormat::Format_YUV422P: + case QVideoSurfaceFormat::Format_YV12: { + static constexpr float pw[] = { 1, 1, 1 }; + planeWidth = pw; + break; + } + case QVideoSurfaceFormat::Format_UYVY: + case QVideoSurfaceFormat::Format_YUYV: { + static constexpr float pw[] = { 1, 1, 0 }; + planeWidth = pw; + break; + } + case QVideoSurfaceFormat::Format_NV12: + case QVideoSurfaceFormat::Format_NV21: + case QVideoSurfaceFormat::Format_P010LE: + case QVideoSurfaceFormat::Format_P010BE: + case QVideoSurfaceFormat::Format_P016LE: + case QVideoSurfaceFormat::Format_P016BE: { + static constexpr float pw[] = { 1, 1, 0 }; + planeWidth = pw; + break; + } + } + // { matrix4x4, colorMatrix, opacity, planeWidth[3] } + QByteArray buf(64*2 + 4 + 3*4, Qt::Uninitialized); + char *data = buf.data(); + memcpy(data, transform.constData(), 64); + memcpy(data + 64, colorMatrix(format.yCbCrColorSpace()).constData(), 64); + memcpy(data + 64 + 64, &opacity, 4); + memcpy(data + 64 + 64 + 4, planeWidth, 3*4); + return buf; +} + +int updateRhiTextures(QVideoFrame frame, QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures) +{ + QVideoSurfaceFormat fmt = frame.surfaceFormat(); + QVideoSurfaceFormat::PixelFormat pixelFormat = fmt.pixelFormat(); + QSize size = fmt.frameSize(); + + const TextureDescription *description = descriptions + pixelFormat; + + QSize planeSizes[TextureDescription::maxPlanes]; + for (int plane = 0; plane < description->nplanes; ++plane) + planeSizes[plane] = QSize(size.width()/description->sizeScale[plane].x, size.height()/description->sizeScale[plane].y); + + if (frame.handleType() == QVideoFrame::RhiTextureHandle) { + for (int plane = 0; plane < description->nplanes; ++plane) { + quint64 nativeTexture = frame.textureHandle(plane); + Q_ASSERT(nativeTexture); + textures[plane] = rhi->newTexture(description->textureFormat[plane], planeSizes[plane], 1, {}); + textures[plane]->createFrom({nativeTexture, 0}); + } + return description->nplanes; + } + + // need to upload textures + bool mapped = frame.map(QVideoFrame::ReadOnly); + if (!mapped) { + qWarning() << "could not map data of QVideoFrame for upload"; + return 0; + } + + Q_ASSERT(frame.planeCount() == description->nplanes); + for (int plane = 0; plane < description->nplanes; ++plane) { + + bool needsRebuild = !textures[plane] || textures[plane]->pixelSize() != planeSizes[plane]; + if (!textures[plane]) + textures[plane] = rhi->newTexture(description->textureFormat[plane], planeSizes[plane], 1, {}); + + if (needsRebuild) { + textures[plane]->setPixelSize(planeSizes[plane]); + bool created = textures[plane]->create(); + if (!created) { + qWarning("Failed to create texture (size %dx%d)", planeSizes[plane].width(), planeSizes[plane].height()); + return 0; + } + } + + QRhiTextureSubresourceUploadDescription subresDesc(frame.bits(plane), frame.bytesPerLine(plane)*planeSizes[plane].height()); + subresDesc.setSourceSize(planeSizes[plane]); + subresDesc.setDestinationTopLeft(QPoint(0, 0)); + QRhiTextureUploadEntry entry(0, 0, subresDesc); + QRhiTextureUploadDescription desc({ entry }); + resourceUpdates->uploadTexture(textures[plane], desc); + } + return description->nplanes; +} + +} diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h b/src/multimedia/video/qvideotexturehelper_p.h index 61aa04873..4b369594a 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture_p.h +++ b/src/multimedia/video/qvideotexturehelper_p.h @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Toolkit. @@ -37,49 +37,41 @@ ** ****************************************************************************/ -#ifndef QSGVIDEONODE_TEXTURE_H -#define QSGVIDEONODE_TEXTURE_H +#ifndef QVIDEOTEXTUREHELPER_H +#define QVIDEOTEXTUREHELPER_H -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qsgvideonode_p.h> -#include <QtMultimedia/qvideosurfaceformat.h> +#include <qvideosurfaceformat.h> +#include <private/qrhi_p.h> QT_BEGIN_NAMESPACE -class QSGVideoMaterial_Texture; +class QVideoFrame; -class QSGVideoNode_Texture : public QSGVideoNode +namespace QVideoTextureHelper { -public: - QSGVideoNode_Texture(const QVideoSurfaceFormat &format); - ~QSGVideoNode_Texture(); - QVideoSurfaceFormat::PixelFormat pixelFormat() const override { - return m_format.pixelFormat(); - } - void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; +struct TextureDescription +{ + static constexpr int maxPlanes = 3; + struct SizeScale { + int x; + int y; + }; -private: - QVideoSurfaceFormat m_format; - QSGVideoMaterial_Texture *m_material; - QVideoFrame m_frame; + int nplanes; + QRhiTexture::Format textureFormat[maxPlanes]; + SizeScale sizeScale[maxPlanes]; }; -class QSGVideoNodeFactory_Texture : public QSGVideoNodeFactoryInterface { -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override; -}; +Q_MULTIMEDIA_EXPORT const TextureDescription *textureDescription(QVideoSurfaceFormat::PixelFormat format); + +Q_MULTIMEDIA_EXPORT QString vertexShaderFileName(QVideoSurfaceFormat::PixelFormat format); +Q_MULTIMEDIA_EXPORT QString fragmentShaderFileName(QVideoSurfaceFormat::PixelFormat format); +Q_MULTIMEDIA_EXPORT QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &transform, float opacity); +Q_MULTIMEDIA_EXPORT int updateRhiTextures(QVideoFrame frame, QRhi *rhi, + QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures); + +} QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/CMakeLists.txt b/src/qtmultimediaquicktools/CMakeLists.txt index 732fab664..96657b9ef 100644 --- a/src/qtmultimediaquicktools/CMakeLists.txt +++ b/src/qtmultimediaquicktools/CMakeLists.txt @@ -11,9 +11,6 @@ qt_add_module(MultimediaQuick qdeclarativevideooutput.cpp qdeclarativevideooutput_p.h qdeclarativevideooutput_render.cpp qdeclarativevideooutput_render_p.h qsgvideonode_p.cpp qsgvideonode_p.h - qsgvideonode_rgb.cpp qsgvideonode_rgb_p.h - qsgvideonode_texture.cpp qsgvideonode_texture_p.h - qsgvideonode_yuv.cpp qsgvideonode_yuv_p.h qsgvideotexture.cpp qsgvideotexture_p.h qtmultimediaquickdefs_p.h PUBLIC_LIBRARIES diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp index a45350fdc..f62a6c891 100644 --- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp +++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp @@ -55,33 +55,10 @@ QT_BEGIN_NAMESPACE Q_DECLARE_LOGGING_CATEGORY(qLcVideo) -Q_GLOBAL_STATIC_WITH_ARGS(QMediaPluginLoader, videoNodeFactoryLoader, - (QSGVideoNodeFactoryInterface_iid, QLatin1String("video/videonode"), Qt::CaseInsensitive)) - QDeclarativeVideoBackend::QDeclarativeVideoBackend(QDeclarativeVideoOutput *parent) : q(parent), m_frameChanged(false) { - // Prioritize the plugin requested by the environment - QString requestedVideoNode = QString::fromLatin1(qgetenv("QT_VIDEONODE")); - - const auto keys = videoNodeFactoryLoader()->keys(); - for (const QString &key : keys) { - QObject *instance = videoNodeFactoryLoader()->instance(key); - QSGVideoNodeFactoryInterface* plugin = qobject_cast<QSGVideoNodeFactoryInterface*>(instance); - if (plugin) { - if (key == requestedVideoNode) - m_videoNodeFactories.prepend(plugin); - else - m_videoNodeFactories.append(plugin); - qCDebug(qLcVideo) << "found videonode plugin" << key << plugin; - } - } - - // Append existing node factories as fallback if we have no plugins - m_videoNodeFactories.append(&m_i420Factory); - m_videoNodeFactories.append(&m_rgbFactory); - m_videoNodeFactories.append(&m_textureFactory); } QDeclarativeVideoBackend::~QDeclarativeVideoBackend() @@ -277,15 +254,8 @@ QSGNode *QDeclarativeVideoBackend::updatePaintNode(QSGNode *oldNode, // Get a node that supports our frame. The surface is irrelevant, our // QSGVideoItemSurface supports (logically) anything. updateGeometry(); - for (QSGVideoNodeFactoryInterface* factory : qAsConst(m_videoNodeFactories)) { - videoNode = factory->createNode(m_surfaceFormat); - if (videoNode) { - qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType() - << " Supported formats for the handle by this node:" - << factory->supportedPixelFormats(m_frame.handleType()); - break; - } - } + videoNode = new QSGVideoNode(m_surfaceFormat); + qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType(); } } diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h index 95b3f1ebd..4fa6e5980 100644 --- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h +++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h @@ -54,9 +54,7 @@ #include <QtQuick/qquickitem.h> #include <QtQuick/qsgnode.h> -#include <private/qsgvideonode_yuv_p.h> -#include <private/qsgvideonode_rgb_p.h> -#include <private/qsgvideonode_texture_p.h> +#include <private/qsgvideonode_p.h> #include <QtCore/qmutex.h> @@ -94,16 +92,12 @@ private: void scheduleDeleteFilterResources(); QDeclarativeVideoOutput *q; - QList<QSGVideoNodeFactoryInterface*> m_videoNodeFactories; mutable QVideoSink *m_sink = nullptr; QVideoSurfaceFormat m_surfaceFormat; QVideoFrame m_frame; QVideoFrame m_frameOnFlush; bool m_frameChanged; - QSGVideoNodeFactory_YUV m_i420Factory; - QSGVideoNodeFactory_RGB m_rgbFactory; - QSGVideoNodeFactory_Texture m_textureFactory; QMutex m_frameMutex; QRectF m_renderedRect; // Destination pixel coordinates, clipped QRectF m_sourceTextureRect; // Source texture coordinates diff --git a/src/qtmultimediaquicktools/qsgvideonode_p.cpp b/src/qtmultimediaquicktools/qsgvideonode_p.cpp index 9cb81902e..30faf95ef 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_p.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_p.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Toolkit. @@ -38,15 +38,13 @@ ****************************************************************************/ #include "qsgvideonode_p.h" +#include <QtQuick/qsgmaterial.h> +#include "qsgvideotexture_p.h" +#include <QtMultimedia/private/qvideotexturehelper_p.h> +#include <qmutex.h> QT_BEGIN_NAMESPACE -QSGVideoNode::QSGVideoNode() - : m_orientation(-1) -{ - setFlag(QSGNode::OwnsGeometry); -} - /* Helpers */ static inline void qSetGeom(QSGGeometry::TexturedPoint2D *v, const QPointF &p) { @@ -124,6 +122,161 @@ void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &tex markDirty(DirtyGeometry); } -QSGVideoNodeFactoryInterface::~QSGVideoNodeFactoryInterface() = default; +class QSGVideoMaterialRhiShader : public QSGMaterialShader +{ +public: + QSGVideoMaterialRhiShader(const QVideoSurfaceFormat &format) + : m_format(format) + { + setShaderFileName(VertexStage, m_format.vertexShaderFileName()); + setShaderFileName(FragmentStage, m_format.fragmentShaderFileName()); + } + + bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, + QSGMaterial *oldMaterial) override; + + void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, + QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; + +protected: + QVideoSurfaceFormat m_format; + float m_planeWidth[3] = {0, 0, 0}; + QMatrix4x4 m_colorMatrix; +}; + +class QSGVideoMaterial : public QSGMaterial +{ +public: + QSGVideoMaterial(const QVideoSurfaceFormat &format); + + [[nodiscard]] QSGMaterialType *type() const override { + static QSGMaterialType type[QVideoSurfaceFormat::NPixelFormats]; + return &type[m_format.pixelFormat()]; + } + + [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { + return new QSGVideoMaterialRhiShader(m_format); + } + + int compare(const QSGMaterial *other) const override { + const QSGVideoMaterial *m = static_cast<const QSGVideoMaterial *>(other); + + qint64 diff = m_textures[0]->comparisonKey() - m->m_textures[0]->comparisonKey(); + if (!diff) + diff = m_textures[1]->comparisonKey() - m->m_textures[1]->comparisonKey(); + if (!diff) + diff = m_textures[2]->comparisonKey() - m->m_textures[2]->comparisonKey(); + + return diff < 0 ? -1 : (diff > 0 ? 1 : 0); + } + + void updateBlending() { + // ### respect video formats with Alpha + setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); + } + + void setCurrentFrame(const QVideoFrame &frame) { + QMutexLocker lock(&m_frameMutex); + m_frame = frame; + m_texturesDirty = true; + } + + void updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates); + + QVideoSurfaceFormat m_format; + float m_planeWidth[3]; + float m_opacity; + + QMutex m_frameMutex; + bool m_texturesDirty = false; + QVideoFrame m_frame; + + QScopedPointer<QSGVideoTexture> m_textures[3]; +}; + +void QSGVideoMaterial::updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) +{ + QMutexLocker locker(&m_frameMutex); + if (!m_texturesDirty) + return; + + // update and upload all textures + QRhiTexture *textures[3] = {}; + for (int i = 0; i < 3; ++i) { + if (m_textures[i].data()) + textures[i] = m_textures[i].data()->rhiTexture(); + else + textures[i] = nullptr; + } + + QVideoTextureHelper::updateRhiTextures(m_frame, rhi, resourceUpdates, textures); + + for (int i = 0; i < 3; ++i) { + if (m_textures[i].data()) + m_textures[i].data()->setRhiTexture(textures[i]); + } +} + + +bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, + QSGMaterial *oldMaterial) +{ + Q_UNUSED(oldMaterial); + + auto m = static_cast<QSGVideoMaterial *>(newMaterial); + + if (!state.isMatrixDirty() && !state.isOpacityDirty()) + return false; + + if (state.isOpacityDirty()) { + m->m_opacity = state.opacity(); + m->updateBlending(); + } + + QByteArray *buf = state.uniformData(); + *buf = m_format.uniformData(state.combinedMatrix(), state.opacity()); + + return true; +} + +void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, + QSGMaterial *newMaterial, QSGMaterial *oldMaterial) +{ + Q_UNUSED(oldMaterial); + if (binding < 1 || binding > 3) + return; + + auto m = static_cast<QSGVideoMaterial *>(newMaterial); + + m->updateTextures(state.rhi(), state.resourceUpdateBatch()); + + *texture = m->m_textures[binding - 1].data(); +} + +QSGVideoMaterial::QSGVideoMaterial(const QVideoSurfaceFormat &format) : + m_format(format), + m_opacity(1.0) +{ + m_textures[0].reset(new QSGVideoTexture); + m_textures[1].reset(new QSGVideoTexture); + m_textures[2].reset(new QSGVideoTexture); + + setFlag(Blending, false); +} + +QSGVideoNode::QSGVideoNode(const QVideoSurfaceFormat &format) + : m_orientation(-1), + m_format(format) +{ + setFlag(QSGNode::OwnsMaterial); + m_material = new QSGVideoMaterial(format); + setMaterial(m_material); +} + +void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame, FrameFlags) +{ + m_material->setCurrentFrame(frame); + markDirty(DirtyMaterial); +} QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_p.h b/src/qtmultimediaquicktools/qsgvideonode_p.h index 7d71cb9f5..36239cdcd 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_p.h +++ b/src/qtmultimediaquicktools/qsgvideonode_p.h @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Toolkit. @@ -60,7 +60,9 @@ QT_BEGIN_NAMESPACE -class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNode : public QSGGeometryNode +class QSGVideoMaterial; + +class QSGVideoNode : public QSGGeometryNode { public: enum FrameFlag { @@ -68,10 +70,12 @@ public: }; Q_DECLARE_FLAGS(FrameFlags, FrameFlag) - QSGVideoNode(); + QSGVideoNode(const QVideoSurfaceFormat &format); - virtual void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) = 0; - virtual QVideoSurfaceFormat::PixelFormat pixelFormat() const = 0; + QVideoSurfaceFormat::PixelFormat pixelFormat() const { + return m_format.pixelFormat(); + } + void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags); void setTexturedRectGeometry(const QRectF &boundingRect, const QRectF &textureRect, int orientation); @@ -79,30 +83,12 @@ private: QRectF m_rect; QRectF m_textureRect; int m_orientation; -}; - -Q_DECLARE_OPERATORS_FOR_FLAGS(QSGVideoNode::FrameFlags) - -class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNodeFactoryInterface -{ -public: - virtual ~QSGVideoNodeFactoryInterface(); - virtual QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const = 0; - virtual QSGVideoNode *createNode(const QVideoSurfaceFormat &format) = 0; + QVideoSurfaceFormat m_format; + QSGVideoMaterial *m_material; }; -#define QSGVideoNodeFactoryInterface_iid "org.qt-project.qt.sgvideonodefactory/5.2" -Q_DECLARE_INTERFACE(QSGVideoNodeFactoryInterface, QSGVideoNodeFactoryInterface_iid) - -class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNodeFactoryPlugin : public QObject, public QSGVideoNodeFactoryInterface -{ - Q_OBJECT - Q_INTERFACES(QSGVideoNodeFactoryInterface) -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override = 0; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override = 0; -}; +Q_DECLARE_OPERATORS_FOR_FLAGS(QSGVideoNode::FrameFlags) QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp b/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp deleted file mode 100644 index 064cf43b0..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qsgvideonode_rgb_p.h" -#include "qsgvideotexture_p.h" -#include <QtQuick/qsgmaterial.h> -#include <QtCore/qmutex.h> - -QT_BEGIN_NAMESPACE - -QList<QVideoSurfaceFormat::PixelFormat> QSGVideoNodeFactory_RGB::supportedPixelFormats( - QVideoFrame::HandleType handleType) const -{ - QList<QVideoSurfaceFormat::PixelFormat> pixelFormats; - - if (handleType == QVideoFrame::NoHandle) { - pixelFormats.append(QVideoSurfaceFormat::Format_RGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_ARGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_ARGB32_Premultiplied); - pixelFormats.append(QVideoSurfaceFormat::Format_BGR32); - pixelFormats.append(QVideoSurfaceFormat::Format_BGRA32); - pixelFormats.append(QVideoSurfaceFormat::Format_RGB565); - } - - return pixelFormats; -} - -QSGVideoNode *QSGVideoNodeFactory_RGB::createNode(const QVideoSurfaceFormat &format) -{ - if (supportedPixelFormats(QVideoFrame::NoHandle).contains(format.pixelFormat())) - return new QSGVideoNode_RGB(format); - - return nullptr; -} - -class QSGVideoMaterialRhiShader_RGB : public QSGMaterialShader -{ -public: - QSGVideoMaterialRhiShader_RGB(const QVideoSurfaceFormat &format) - : m_format(format) - { - setShaderFileName(VertexStage, format.vertexShaderFileName()); - setShaderFileName(FragmentStage, format.fragmentShaderFileName()); - } - - bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) override; - - void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; - QVideoSurfaceFormat m_format; -}; - -class QSGVideoMaterial_RGB : public QSGMaterial -{ -public: - QSGVideoMaterial_RGB(const QVideoSurfaceFormat &format) : - m_format(format), - m_opacity(1.0) - { - setFlag(Blending, false); - m_texture.reset(new QSGVideoTexture); - } - - [[nodiscard]] QSGMaterialType *type() const override { - static QSGMaterialType normalType; - return &normalType; - } - - [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { - return new QSGVideoMaterialRhiShader_RGB(m_format); - } - - int compare(const QSGMaterial *other) const override { - const QSGVideoMaterial_RGB *m = static_cast<const QSGVideoMaterial_RGB *>(other); - - const qint64 diff = m_texture->comparisonKey() - m->m_texture->comparisonKey(); - return diff < 0 ? -1 : (diff > 0 ? 1 : 0); - } - - void updateBlending() { - setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); - } - - void setVideoFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; - } - - QVideoFrame m_frame; - QMutex m_frameMutex; - QSize m_textureSize; - QVideoSurfaceFormat m_format; - float m_opacity; - QScopedPointer<QSGVideoTexture> m_texture; -}; - -bool QSGVideoMaterialRhiShader_RGB::updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - if (!state.isMatrixDirty() && !state.isOpacityDirty()) - return false; - - if (state.isOpacityDirty()) { - auto m = static_cast<QSGVideoMaterial_RGB *>(newMaterial); - m->m_opacity = state.opacity(); - m->updateBlending(); - } - - QByteArray *buf = state.uniformData(); - *buf = m_format.uniformData(state.combinedMatrix(), state.opacity()); - - return true; -} - -void QSGVideoMaterialRhiShader_RGB::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - if (binding < 1) - return; - - auto m = static_cast<QSGVideoMaterial_RGB *>(newMaterial); - - m->m_frameMutex.lock(); - auto frame = m->m_frame; - - if (frame.pixelFormat() == QVideoSurfaceFormat::Format_RGB565) // Format_RGB565 requires GL_UNSIGNED_SHORT_5_6_5 - frame = frame.toImage().convertToFormat(QImage::Format_RGBA8888_Premultiplied); - - auto format = QRhiTexture::RGBA8; - if (frame.pixelFormat() == QVideoSurfaceFormat::Format_RGB32 - || frame.pixelFormat() == QVideoSurfaceFormat::Format_ARGB32 - || frame.pixelFormat() == QVideoSurfaceFormat::Format_ARGB32_Premultiplied) - { - format = QRhiTexture::BGRA8; - } - - if (frame.isValid() && frame.map(QVideoFrame::ReadOnly)) { - m->m_texture->setData(format, frame.size(), frame.bits(), frame.bytesPerLine() * frame.height()); - frame.unmap(); - } - m->m_frameMutex.unlock(); - - m->m_texture->commitTextureOperations(state.rhi(), state.resourceUpdateBatch()); - *texture = m->m_texture.data(); -} - -QSGVideoNode_RGB::QSGVideoNode_RGB(const QVideoSurfaceFormat &format) : - m_format(format) -{ - setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_RGB(format); - setMaterial(m_material); -} - -QSGVideoNode_RGB::~QSGVideoNode_RGB() = default; - -void QSGVideoNode_RGB::setCurrentFrame(const QVideoFrame &frame, FrameFlags) -{ - m_material->setVideoFrame(frame); - markDirty(DirtyMaterial); -} - -QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb_p.h b/src/qtmultimediaquicktools/qsgvideonode_rgb_p.h deleted file mode 100644 index b42509c1f..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_rgb_p.h +++ /dev/null @@ -1,86 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QSGVIDEONODE_RGB_H -#define QSGVIDEONODE_RGB_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qsgvideonode_p.h> -#include <QtMultimedia/qvideosurfaceformat.h> - -QT_BEGIN_NAMESPACE - -class QSGVideoMaterial_RGB; - -class QSGVideoNode_RGB : public QSGVideoNode -{ -public: - QSGVideoNode_RGB(const QVideoSurfaceFormat &format); - ~QSGVideoNode_RGB(); - - QVideoSurfaceFormat::PixelFormat pixelFormat() const override { - return m_format.pixelFormat(); - } - void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; - -private: - QVideoSurfaceFormat m_format; - QSGVideoMaterial_RGB *m_material; - QVideoFrame m_frame; -}; - -class QSGVideoNodeFactory_RGB : public QSGVideoNodeFactoryInterface { -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override; -}; - -QT_END_NAMESPACE - -#endif // QSGVIDEONODE_RGB_H diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp deleted file mode 100644 index 8f363cd46..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp +++ /dev/null @@ -1,212 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qsgvideonode_texture_p.h" -#include "qsgvideotexture_p.h" -#include <private/qsgrhisupport_p.h> -#include <QtQuick/qsgmaterial.h> -#include <QtCore/qmutex.h> -#include <QtMultimedia/private/qtmultimediaglobal_p.h> - -QT_BEGIN_NAMESPACE - -QList<QVideoSurfaceFormat::PixelFormat> QSGVideoNodeFactory_Texture::supportedPixelFormats( - QVideoFrame::HandleType) const -{ - QList<QVideoSurfaceFormat::PixelFormat> pixelFormats; - - pixelFormats.append(QVideoSurfaceFormat::Format_RGB565); - pixelFormats.append(QVideoSurfaceFormat::Format_RGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_ARGB32); - pixelFormats.append(QVideoSurfaceFormat::Format_BGR32); - pixelFormats.append(QVideoSurfaceFormat::Format_BGRA32); -#if !QT_CONFIG(gpu_vivante) - pixelFormats.append(QVideoSurfaceFormat::Format_ABGR32); -#endif - - return pixelFormats; -} - -QSGVideoNode *QSGVideoNodeFactory_Texture::createNode(const QVideoSurfaceFormat &format) -{ - if (supportedPixelFormats(QVideoFrame::NoHandle).contains(format.pixelFormat())) - return new QSGVideoNode_Texture(format); - - return nullptr; -} - -class QSGVideoMaterialRhiShader_Texture : public QSGMaterialShader -{ -public: - QSGVideoMaterialRhiShader_Texture() - { - setShaderFileName(VertexStage, QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb")); - setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb")); - } - - bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) override; - - void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; -}; - -class QSGVideoMaterialRhiShader_Texture_swizzle : public QSGVideoMaterialRhiShader_Texture -{ -public: - QSGVideoMaterialRhiShader_Texture_swizzle() - { - setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimedia/shaders/bgra.frag.qsb")); - } -}; - -class QSGVideoMaterial_Texture : public QSGMaterial -{ -public: - QSGVideoMaterial_Texture(const QVideoSurfaceFormat &format) : - m_format(format), - m_opacity(1.0) - { - setFlag(Blending, false); - m_texture.reset(new QSGVideoTexture); - } - - ~QSGVideoMaterial_Texture() override = default; - - [[nodiscard]] QSGMaterialType *type() const override { - static QSGMaterialType normalType, swizzleType; - return needsSwizzling() ? &swizzleType : &normalType; - } - - [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { - return needsSwizzling() ? new QSGVideoMaterialRhiShader_Texture_swizzle - : new QSGVideoMaterialRhiShader_Texture; - } - - int compare(const QSGMaterial *other) const override { - const QSGVideoMaterial_Texture *m = static_cast<const QSGVideoMaterial_Texture *>(other); - - const qint64 diff = m_texture->comparisonKey() - m->m_texture->comparisonKey(); - return diff < 0 ? -1 : (diff > 0 ? 1 : 0); - } - - void updateBlending() { - setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); - } - - void setVideoFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; - } - - QVideoFrame m_frame; - QMutex m_frameMutex; - QVideoSurfaceFormat m_format; - quint64 m_textureId; - float m_opacity; - QScopedPointer<QSGVideoTexture> m_texture; - -private: - [[nodiscard]] bool needsSwizzling() const { - return m_format.pixelFormat() == QVideoSurfaceFormat::Format_RGB32 - || m_format.pixelFormat() == QVideoSurfaceFormat::Format_ARGB32; - } -}; - -bool QSGVideoMaterialRhiShader_Texture::updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - bool changed = false; - QByteArray *buf = state.uniformData(); - - if (state.isMatrixDirty()) { - memcpy(buf->data(), state.combinedMatrix().constData(), 64); - changed = true; - } - - if (state.isOpacityDirty()) { - auto m = static_cast<QSGVideoMaterial_Texture *>(newMaterial); - m->m_opacity = state.opacity(); - m->updateBlending(); - memcpy(buf->data() + 64, &m->m_opacity, 4); - changed = true; - } - - return changed; -} - -void QSGVideoMaterialRhiShader_Texture::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - if (binding < 1) - return; - - auto m = static_cast<QSGVideoMaterial_Texture *>(newMaterial); - - m->m_frameMutex.lock(); - auto size = m->m_frame.size(); - if (m->m_frame.isValid()) - m->m_textureId = m->m_frame.textureHandle(0); - m->m_frameMutex.unlock(); - - m->m_texture->setNativeObject(m->m_textureId, size); - m->m_texture->commitTextureOperations(state.rhi(), state.resourceUpdateBatch()); - *texture = m->m_texture.data(); -} - -QSGVideoNode_Texture::QSGVideoNode_Texture(const QVideoSurfaceFormat &format) : - m_format(format) -{ - setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_Texture(format); - setMaterial(m_material); -} - -QSGVideoNode_Texture::~QSGVideoNode_Texture() = default; - -void QSGVideoNode_Texture::setCurrentFrame(const QVideoFrame &frame, FrameFlags) -{ - m_material->setVideoFrame(frame); - markDirty(DirtyMaterial); -} - -QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp deleted file mode 100644 index d305e52a7..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp +++ /dev/null @@ -1,424 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qsgvideonode_yuv_p.h" -#include "qsgvideotexture_p.h" -#include <QtCore/qmutex.h> -#include <QtQuick/qsgmaterial.h> - -QT_BEGIN_NAMESPACE - -QList<QVideoSurfaceFormat::PixelFormat> QSGVideoNodeFactory_YUV::supportedPixelFormats( - QVideoFrame::HandleType) const -{ - QList<QVideoSurfaceFormat::PixelFormat> formats; - - formats << QVideoSurfaceFormat::Format_YUV420P << QVideoSurfaceFormat::Format_YV12 << QVideoSurfaceFormat::Format_YUV422P - << QVideoSurfaceFormat::Format_NV12 << QVideoSurfaceFormat::Format_NV21 - << QVideoSurfaceFormat::Format_UYVY << QVideoSurfaceFormat::Format_YUYV - << QVideoSurfaceFormat::Format_P010BE << QVideoSurfaceFormat::Format_P010LE - << QVideoSurfaceFormat::Format_P016BE << QVideoSurfaceFormat::Format_P016LE - ; - - return formats; -} - -QSGVideoNode *QSGVideoNodeFactory_YUV::createNode(const QVideoSurfaceFormat &format) -{ - if (supportedPixelFormats(QVideoFrame::NoHandle).contains(format.pixelFormat())) - return new QSGVideoNode_YUV(format); - - return nullptr; -} - -class QSGVideoMaterialRhiShader_YUV : public QSGMaterialShader -{ -public: - QSGVideoMaterialRhiShader_YUV(const QVideoSurfaceFormat &format) - : m_format(format) - { - setShaderFileName(VertexStage, m_format.vertexShaderFileName()); - setShaderFileName(FragmentStage, m_format.fragmentShaderFileName()); - } - - bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) override; - - void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; - - virtual void mapFrame(QSGVideoMaterial_YUV *) = 0; - -protected: - QVideoSurfaceFormat m_format; - float m_planeWidth[3] = {0, 0, 0}; - QMatrix4x4 m_colorMatrix; -}; - -class QSGVideoMaterialRhiShader_UYVY : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_UYVY(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterialRhiShader_YUV_YV : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_YUV_YV(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterialRhiShader_NV12 : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_NV12(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterialRhiShader_P010 : public QSGVideoMaterialRhiShader_YUV -{ -public: - QSGVideoMaterialRhiShader_P010(const QVideoSurfaceFormat &format) - : QSGVideoMaterialRhiShader_YUV(format) - {} - void mapFrame(QSGVideoMaterial_YUV *m) override; -}; - -class QSGVideoMaterial_YUV : public QSGMaterial -{ -public: - QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format); - - [[nodiscard]] QSGMaterialType *type() const override { - static QSGMaterialType biPlanarType, biPlanarSwizzleType, triPlanarType, uyvyType, yuyvType; - - switch (m_format.pixelFormat()) { - case QVideoSurfaceFormat::Format_NV12: - case QVideoSurfaceFormat::Format_P010BE: - case QVideoSurfaceFormat::Format_P010LE: - case QVideoSurfaceFormat::Format_P016LE: - case QVideoSurfaceFormat::Format_P016BE: - return &biPlanarType; - case QVideoSurfaceFormat::Format_NV21: - return &biPlanarSwizzleType; - case QVideoSurfaceFormat::Format_UYVY: - return &uyvyType; - case QVideoSurfaceFormat::Format_YUYV: - return &yuyvType; - default: // Currently: YUV420P, YUV422P and YV12 - return &triPlanarType; - } - } - - [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { - switch (m_format.pixelFormat()) { - case QVideoSurfaceFormat::Format_NV12: - case QVideoSurfaceFormat::Format_NV21: - return new QSGVideoMaterialRhiShader_NV12(m_format); - case QVideoSurfaceFormat::Format_UYVY: - case QVideoSurfaceFormat::Format_YUYV: - return new QSGVideoMaterialRhiShader_UYVY(m_format); - case QVideoSurfaceFormat::Format_P010LE: - case QVideoSurfaceFormat::Format_P016LE: - case QVideoSurfaceFormat::Format_P010BE: - case QVideoSurfaceFormat::Format_P016BE: - return new QSGVideoMaterialRhiShader_P010(m_format); - default: // Currently: YUV420P, YUV422P and YV12 - return new QSGVideoMaterialRhiShader_YUV_YV(m_format); - } - } - - int compare(const QSGMaterial *other) const override { - const QSGVideoMaterial_YUV *m = static_cast<const QSGVideoMaterial_YUV *>(other); - - qint64 diff = m_textures[0]->comparisonKey() - m->m_textures[0]->comparisonKey(); - if (!diff) - diff = m_textures[1]->comparisonKey() - m->m_textures[1]->comparisonKey(); - if (!diff) - diff = m_textures[2]->comparisonKey() - m->m_textures[2]->comparisonKey(); - - return diff < 0 ? -1 : (diff > 0 ? 1 : 0); - } - - void updateBlending() { - setFlag(Blending, !qFuzzyCompare(m_opacity, float(1.0))); - } - - void setCurrentFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; - } - - QVideoSurfaceFormat m_format; - float m_planeWidth[3]; - float m_opacity; - QMatrix4x4 m_colorMatrix; - QVideoFrame m_frame; - QMutex m_frameMutex; - QScopedPointer<QSGVideoTexture> m_textures[3]; -}; - -bool QSGVideoMaterialRhiShader_YUV::updateUniformData(RenderState &state, QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - - auto m = static_cast<QSGVideoMaterial_YUV *>(newMaterial); - m->m_frameMutex.lock(); - mapFrame(m); - m->m_frameMutex.unlock(); - - if (!state.isMatrixDirty() && !state.isOpacityDirty() && m->m_colorMatrix == m_colorMatrix) - return false; - - if (state.isOpacityDirty()) { - m->m_opacity = state.opacity(); - m->updateBlending(); - } - - m_colorMatrix = m->m_colorMatrix; - - QByteArray *buf = state.uniformData(); - *buf = m_format.uniformData(state.combinedMatrix(), state.opacity()); - - return true; -} - -void QSGVideoMaterialRhiShader_YUV::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, - QSGMaterial *newMaterial, QSGMaterial *oldMaterial) -{ - Q_UNUSED(oldMaterial); - if (binding < 1 || binding > 3) - return; - - auto m = static_cast<QSGVideoMaterial_YUV *>(newMaterial); - *texture = m->m_textures[binding - 1].data(); - (*texture)->commitTextureOperations(state.rhi(), state.resourceUpdateBatch()); -} - -void QSGVideoMaterialRhiShader_UYVY::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid() || !m->m_frame.map(QVideoFrame::ReadOnly)) - return; - - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - - m->m_planeWidth[0] = 1; - m->m_planeWidth[1] = 1; - - // Either r,b (YUYV) or g,a (UYVY) values are used as source of UV. - // Additionally U and V are set per 2 pixels hence only 1/2 of image width is used. - m->m_textures[0]->setData(QRhiTexture::RG8, m->m_frame.size(), - m->m_frame.bits(), m->m_frame.bytesPerLine() * fh); - m->m_textures[1]->setData(QRhiTexture::RGBA8, QSize(fw / 2, fh), - m->m_frame.bits(), m->m_frame.bytesPerLine() * fh); - - m->m_frame.unmap(); -} - -void QSGVideoMaterialRhiShader_YUV_YV::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid() || !m->m_frame.map(QVideoFrame::ReadOnly)) - return; - - int y = 0; - int u = m->m_frame.pixelFormat() == QVideoSurfaceFormat::Format_YV12 ? 2 : 1; - int v = m->m_frame.pixelFormat() == QVideoSurfaceFormat::Format_YV12 ? 1 : 2; - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - int uvHeight = m->m_frame.pixelFormat() == QVideoSurfaceFormat::Format_YUV422P ? fh : fh / 2; - - m->m_planeWidth[0] = float(fw) / m->m_frame.bytesPerLine(y); - m->m_planeWidth[1] = m->m_planeWidth[2] = float(fw) / (2 * m->m_frame.bytesPerLine(u)); - - m->m_textures[0]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(y), fh), - m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh); - m->m_textures[1]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(u), uvHeight), - m->m_frame.bits(u), m->m_frame.bytesPerLine(u) * uvHeight); - m->m_textures[2]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(v), uvHeight), - m->m_frame.bits(v), m->m_frame.bytesPerLine(v) * uvHeight); - - m->m_frame.unmap(); -} - -void QSGVideoMaterialRhiShader_NV12::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid()) - return; - - if (m->m_frame.handleType() == QVideoFrame::RhiTextureHandle) { - m->m_planeWidth[0] = m->m_planeWidth[1] = 1; - quint64 textures[2] = { m->m_frame.textureHandle(0), m->m_frame.textureHandle(1) }; - if (textures[0] && textures[1]) { - auto w = m->m_frame.size().width(); - auto h = m->m_frame.size().height(); - m->m_textures[0]->setNativeObject(textures[0], {w, h}, QRhiTexture::R8); - m->m_textures[1]->setNativeObject(textures[1], {w / 2, h / 2}, QRhiTexture::RG8); - } else { - qWarning() << "NV12/NV21 requires 2 textures"; - } - - return; - } - - if (!m->m_frame.map(QVideoFrame::ReadOnly)) { - qWarning()<< "NV12: Couldn't map frame"; - m->m_textures[0]->setData(QRhiTexture::RG8, QSize(1, 1), (const uchar *)"\0\0", 2); - m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(1, 1), (const uchar *)"\0\0\0\0", 4); - return; - } - - int y = 0; - int uv = 1; - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - - m->m_planeWidth[0] = m->m_planeWidth[1] = qreal(fw) / m->m_frame.bytesPerLine(y); - - m->m_textures[0]->setData(QRhiTexture::R8, m->m_frame.size(), - m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh); - m->m_textures[1]->setData(QRhiTexture::RG8, QSize(m->m_frame.bytesPerLine(uv) / 2 , fh / 2), - m->m_frame.bits(uv), m->m_frame.bytesPerLine(uv) * fh / 2); - - m->m_frame.unmap(); -} - -void QSGVideoMaterialRhiShader_P010::mapFrame(QSGVideoMaterial_YUV *m) -{ - if (!m->m_frame.isValid()) - return; - - if (m->m_frame.handleType() == QVideoFrame::RhiTextureHandle) { - m->m_planeWidth[0] = m->m_planeWidth[1] = 1; - quint64 textures[2] = { m->m_frame.textureHandle(0), m->m_frame.textureHandle(1) }; - if (textures[0] && textures[1]) { - auto w = m->m_frame.size().width(); - auto h = m->m_frame.size().height(); - m->m_textures[0]->setNativeObject(textures[0], {w, h}, QRhiTexture::RG8); - m->m_textures[1]->setNativeObject(textures[1], {w / 2, h / 2}, QRhiTexture::BGRA8); - } else { - qWarning() << "P010/P016 requires 2 textures"; - } - - return; - } - - if (!m->m_frame.map(QVideoFrame::ReadOnly)) { - qWarning()<< "NV12: Couldn't map frame"; - m->m_textures[0]->setData(QRhiTexture::RG8, QSize(1, 1), (const uchar *)"\0\0", 2); - m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(1, 1), (const uchar *)"\0\0\0\0", 4); - return; - } - - int y = 0; - int uv = 1; - int fw = m->m_frame.width(); - int fh = m->m_frame.height(); - - m->m_planeWidth[0] = m->m_planeWidth[1] = qreal(fw) / m->m_frame.bytesPerLine(y); - - // Use RG8 and BGRA8 textures for Y and UV. This allows us to pick up the 8 most significant - // bits of the values easily. Since our target surface is not HDR capable this will do for now. - m->m_textures[0]->setData(QRhiTexture::RG8, m->m_frame.size(), - m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh); - m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(m->m_frame.bytesPerLine(uv) / 4 , fh / 2), - m->m_frame.bits(uv), m->m_frame.bytesPerLine(uv) * fh / 2); - - m->m_frame.unmap(); -} - -QSGVideoMaterial_YUV::QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format) : - m_format(format), - m_opacity(1.0) -{ - m_textures[0].reset(new QSGVideoTexture); - m_textures[1].reset(new QSGVideoTexture); - m_textures[2].reset(new QSGVideoTexture); - - switch (format.yCbCrColorSpace()) { - case QVideoSurfaceFormat::YCbCr_JPEG: - m_colorMatrix = QMatrix4x4( - 1.0f, 0.000f, 1.402f, -0.701f, - 1.0f, -0.344f, -0.714f, 0.529f, - 1.0f, 1.772f, 0.000f, -0.886f, - 0.0f, 0.000f, 0.000f, 1.0000f); - break; - case QVideoSurfaceFormat::YCbCr_BT709: - case QVideoSurfaceFormat::YCbCr_xvYCC709: - m_colorMatrix = QMatrix4x4( - 1.164f, 0.000f, 1.793f, -0.5727f, - 1.164f, -0.534f, -0.213f, 0.3007f, - 1.164f, 2.115f, 0.000f, -1.1302f, - 0.0f, 0.000f, 0.000f, 1.0000f); - break; - default: //BT 601: - m_colorMatrix = QMatrix4x4( - 1.164f, 0.000f, 1.596f, -0.8708f, - 1.164f, -0.392f, -0.813f, 0.5296f, - 1.164f, 2.017f, 0.000f, -1.081f, - 0.0f, 0.000f, 0.000f, 1.0000f); - } - - setFlag(Blending, false); -} - -QSGVideoNode_YUV::QSGVideoNode_YUV(const QVideoSurfaceFormat &format) : - m_format(format) -{ - setFlag(QSGNode::OwnsMaterial); - m_material = new QSGVideoMaterial_YUV(format); - setMaterial(m_material); -} - -QSGVideoNode_YUV::~QSGVideoNode_YUV() = default; - -void QSGVideoNode_YUV::setCurrentFrame(const QVideoFrame &frame, FrameFlags) -{ - m_material->setCurrentFrame(frame); - markDirty(DirtyMaterial); -} - -QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv_p.h b/src/qtmultimediaquicktools/qsgvideonode_yuv_p.h deleted file mode 100644 index 55b3bc764..000000000 --- a/src/qtmultimediaquicktools/qsgvideonode_yuv_p.h +++ /dev/null @@ -1,86 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QSGVIDEONODE_YUV_H -#define QSGVIDEONODE_YUV_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qsgvideonode_p.h> -#include <QtMultimedia/qvideosurfaceformat.h> - -QT_BEGIN_NAMESPACE - -class QSGVideoMaterial_YUV; -class QSGVideoNode_YUV : public QSGVideoNode -{ -public: - QSGVideoNode_YUV(const QVideoSurfaceFormat &format); - ~QSGVideoNode_YUV(); - - QVideoSurfaceFormat::PixelFormat pixelFormat() const override { - return m_format.pixelFormat(); - } - void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) override; - -private: - void bindTexture(int id, int unit, int w, int h, const uchar *bits); - - QVideoSurfaceFormat m_format; - QSGVideoMaterial_YUV *m_material; -}; - -class QSGVideoNodeFactory_YUV : public QSGVideoNodeFactoryInterface { -public: - QList<QVideoSurfaceFormat::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override; - QSGVideoNode *createNode(const QVideoSurfaceFormat &format) override; -}; - -QT_END_NAMESPACE - -#endif // QSGVIDEONODE_YUV_H diff --git a/src/qtmultimediaquicktools/qsgvideotexture.cpp b/src/qtmultimediaquicktools/qsgvideotexture.cpp index 18c060de7..40a8f71e1 100644 --- a/src/qtmultimediaquicktools/qsgvideotexture.cpp +++ b/src/qtmultimediaquicktools/qsgvideotexture.cpp @@ -161,4 +161,9 @@ void QSGVideoTexture::commitTextureOperations(QRhi *rhi, QRhiResourceUpdateBatch d_func()->updateRhiTexture(rhi, resourceUpdates); } +void QSGVideoTexture::setRhiTexture(QRhiTexture *texture) +{ + d_func()->m_texture.reset(texture); +} + QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qsgvideotexture_p.h b/src/qtmultimediaquicktools/qsgvideotexture_p.h index 221b891e3..869cae4ba 100644 --- a/src/qtmultimediaquicktools/qsgvideotexture_p.h +++ b/src/qtmultimediaquicktools/qsgvideotexture_p.h @@ -72,6 +72,7 @@ public: bool hasAlphaChannel() const override; bool hasMipmaps() const override; void commitTextureOperations(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) override; + void setRhiTexture(QRhiTexture *texture); void setData(QRhiTexture::Format f, const QSize &s, const uchar *data, int bytes); void setNativeObject(quint64 obj, const QSize &s, QRhiTexture::Format f = QRhiTexture::RGBA8); |