Move the shaders from the quick tools into QtMultimedia

The shader layout is pretty generic, and QVideoSurfaceFormat
can give us both the shaders and uniform data that we need
to render the video.

All the user needs to do then is to provide geometry with a
2D texture coordinate to render the video onto any surface.

Change-Id: I95006eaf4a6f039006b8a5da0f53143da03e1ecb
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

2 files changed, 223 insertions, 0 deletions

diff --git a/src/multimedia/video/qvideosurfaceformat.cpp b/src/multimedia/video/qvideosurfaceformat.cpp
index b29da03e7..774a72c30 100644
--- a/src/multimedia/video/qvideosurfaceformat.cpp
+++ b/src/multimedia/video/qvideosurfaceformat.cpp
@@ -44,9 +44,14 @@
 #include <qmetatype.h>
 #include <qpair.h>
 #include <qvariant.h>
+#include <qmatrix4x4.h>
 
 QT_BEGIN_NAMESPACE
 
+static void initResource() {
+    Q_INIT_RESOURCE(qtmultimedia);
+}
+
 class QVideoSurfaceFormatPrivate : public QSharedData
 {
 public:
@@ -162,6 +167,7 @@ public:
 QVideoSurfaceFormat::QVideoSurfaceFormat()
     : d(new QVideoSurfaceFormatPrivate)
 {
+    initResource();
 }
 
 /*!
@@ -430,6 +436,218 @@ QSize QVideoSurfaceFormat::sizeHint() const
     return d->viewport.size();
 }
 
+QString QVideoSurfaceFormat::vertexShaderFileName() const
+{
+    switch (d->pixelFormat) {
+    case Format_Invalid:
+    case Format_Jpeg:
+
+    case Format_RGB24:
+    case Format_RGB565:
+    case Format_RGB555:
+    case Format_ARGB8565_Premultiplied:
+    case Format_BGR24:
+    case Format_BGR565:
+    case Format_BGR555:
+    case Format_BGRA5658_Premultiplied:
+
+    case Format_Y8:
+    case Format_Y16:
+
+    case Format_AYUV444:
+    case Format_AYUV444_Premultiplied:
+    case Format_YUV444:
+
+    case Format_IMC1:
+    case Format_IMC2:
+    case Format_IMC3:
+    case Format_IMC4:
+        return QString();
+    case Format_ARGB32:
+    case Format_ARGB32_Premultiplied:
+    case Format_RGB32:
+    case Format_BGRA32:
+    case Format_BGRA32_Premultiplied:
+    case Format_ABGR32:
+    case Format_BGR32:
+        return QStringLiteral(":/qtmultimedia/shaders/rgba.vert.qsb");
+    case Format_YUV420P:
+    case Format_YUV422P:
+    case Format_YV12:
+    case Format_UYVY:
+    case Format_YUYV:
+    case Format_NV12:
+    case Format_NV21:
+    case Format_P010LE:
+    case Format_P010BE:
+    case Format_P016LE:
+    case Format_P016BE:
+        return QStringLiteral(":/qtmultimedia/shaders/yuv.vert.qsb");
+    }
+}
+
+QString QVideoSurfaceFormat::pixelShaderFileName() const
+{
+    switch (d->pixelFormat) {
+    case Format_Invalid:
+    case Format_Jpeg:
+
+    case Format_RGB24:
+    case Format_RGB565:
+    case Format_RGB555:
+    case Format_ARGB8565_Premultiplied:
+    case Format_BGR24:
+    case Format_BGR565:
+    case Format_BGR555:
+    case Format_BGRA5658_Premultiplied:
+
+    case Format_Y8:
+    case Format_Y16:
+
+    case Format_AYUV444:
+    case Format_AYUV444_Premultiplied:
+    case Format_YUV444:
+
+    case Format_IMC1:
+    case Format_IMC2:
+    case Format_IMC3:
+    case Format_IMC4:
+        return QString();
+    case Format_ARGB32:
+    case Format_ARGB32_Premultiplied:
+    case Format_RGB32:
+    case Format_BGRA32:
+    case Format_BGRA32_Premultiplied:
+    case Format_ABGR32:
+    case Format_BGR32:
+        return QStringLiteral(":/qtmultimedia/shaders/rgba.frag.qsb");
+    case Format_YUV420P:
+    case Format_YUV422P:
+    case Format_YV12:
+        return QStringLiteral(":/qtmultimedia/shaders/yuv_yv.frag.qsb");
+    case Format_UYVY:
+        return QStringLiteral(":/qtmultimedia/shaders/uyvy.frag.qsb");
+    case Format_YUYV:
+        return QStringLiteral(":/qtmultimedia/shaders/yuyv.frag.qsb");
+    case Format_NV12:
+        return QStringLiteral(":/qtmultimedia/shaders/nv12.frag.qsb");
+    case Format_NV21:
+        return QStringLiteral(":/qtmultimedia/shaders/nv21.frag.qsb");
+    case Format_P010LE:
+    case Format_P016LE:
+        return QStringLiteral(":/qtmultimedia/shaders/p010le.frag.qsb");
+    case Format_P010BE:
+    case Format_P016BE:
+        return QStringLiteral(":/qtmultimedia/shaders/p010be.frag.qsb");
+    }
+}
+
+static QMatrix4x4 colorMatrix(QVideoSurfaceFormat::YCbCrColorSpace colorSpace)
+{
+    switch (colorSpace) {
+    case QVideoSurfaceFormat::YCbCr_JPEG:
+        return QMatrix4x4(
+            1.0f,  0.000f,  1.402f, -0.701f,
+            1.0f, -0.344f, -0.714f,  0.529f,
+            1.0f,  1.772f,  0.000f, -0.886f,
+            0.0f,  0.000f,  0.000f,  1.0000f);
+    case QVideoSurfaceFormat::YCbCr_BT709:
+    case QVideoSurfaceFormat::YCbCr_xvYCC709:
+        return QMatrix4x4(
+            1.164f,  0.000f,  1.793f, -0.5727f,
+            1.164f, -0.534f, -0.213f,  0.3007f,
+            1.164f,  2.115f,  0.000f, -1.1302f,
+            0.0f,    0.000f,  0.000f,  1.0000f);
+    default: //BT 601:
+        return QMatrix4x4(
+            1.164f,  0.000f,  1.596f, -0.8708f,
+            1.164f, -0.392f, -0.813f,  0.5296f,
+            1.164f,  2.017f,  0.000f, -1.081f,
+            0.0f,    0.000f,  0.000f,  1.0000f);
+    }
+}
+
+QByteArray QVideoSurfaceFormat::uniformData(const QMatrix4x4 &transform, float opacity) const
+{
+    static constexpr float pw[3] = {};
+    const float *planeWidth = pw;
+
+    switch (d->pixelFormat) {
+    case Format_Invalid:
+    case Format_Jpeg:
+
+    case Format_RGB24:
+    case Format_RGB565:
+    case Format_RGB555:
+    case Format_ARGB8565_Premultiplied:
+    case Format_BGR24:
+    case Format_BGR565:
+    case Format_BGR555:
+    case Format_BGRA5658_Premultiplied:
+
+    case Format_Y8:
+    case Format_Y16:
+
+    case Format_AYUV444:
+    case Format_AYUV444_Premultiplied:
+    case Format_YUV444:
+
+    case Format_IMC1:
+    case Format_IMC2:
+    case Format_IMC3:
+    case Format_IMC4:
+        return QByteArray();
+    case Format_ARGB32:
+    case Format_ARGB32_Premultiplied:
+    case Format_RGB32:
+    case Format_BGRA32:
+    case Format_BGRA32_Premultiplied:
+    case Format_ABGR32:
+    case Format_BGR32: {
+        // { matrix4x4, opacity }
+        QByteArray buf(16*4 + 4, Qt::Uninitialized);
+        char *data = buf.data();
+        memcpy(data, transform.constData(), 64);
+        memcpy(data + 64, colorMatrix(d->ycbcrColorSpace).constData(), 64);
+        memcpy(data + 64 + 64, &opacity, 4);
+        memcpy(data + 64 + 64, &opacity, 4);
+        memcpy(data + 64 + 64, &opacity, 4);
+        memcpy(data + 64 + 64, &opacity, 4);
+        return buf;
+    }
+    case Format_YUV420P:
+    case Format_YUV422P:
+    case Format_YV12: {
+        static constexpr float pw[] = { 1, 1, 0 };
+        planeWidth = pw;
+        break;
+    }
+    case Format_UYVY:
+    case Format_YUYV: {
+        static constexpr float pw[] = { 1, 1, 0 };
+        planeWidth = pw;
+        break;
+    }
+    case Format_NV12:
+    case Format_NV21:
+    case Format_P010LE:
+    case Format_P010BE:
+    case Format_P016LE:
+    case Format_P016BE: {
+        static constexpr float pw[] = { 1, 1, 0 };
+        planeWidth = pw;
+        break;
+    }
+    }
+    // { matrix4x4, colorMatrix, opacity, planeWidth[3] }
+    QByteArray buf(64*2 + 4 + 3*4, Qt::Uninitialized);
+    char *data = buf.data();
+    memcpy(data, transform.constData(), 64);
+    memcpy(data + 64, &opacity, 4);
+    memcpy(data + 64 + 4, planeWidth, 3*4);
+    return buf;
+}
+
 
 /*!
     Returns a video pixel format equivalent to an image \a format.  If there is no equivalent
diff --git a/src/multimedia/video/qvideosurfaceformat.h b/src/multimedia/video/qvideosurfaceformat.h
index 837d9545a..c81416945 100644
--- a/src/multimedia/video/qvideosurfaceformat.h
+++ b/src/multimedia/video/qvideosurfaceformat.h
@@ -54,6 +54,7 @@ QT_BEGIN_NAMESPACE
 class QDebug;
 
 class QVideoSurfaceFormatPrivate;
+class QMatrix4x4;
 
 class Q_MULTIMEDIA_EXPORT QVideoSurfaceFormat
 {
@@ -163,6 +164,10 @@ public:
 
     QSize sizeHint() const;
 
+    QString vertexShaderFileName() const;
+    QString pixelShaderFileName() const;
+    QByteArray uniformData(const QMatrix4x4 &transform, float opacity) const;
+
     static PixelFormat pixelFormatFromImageFormat(QImage::Format format);
     static QImage::Format imageFormatFromPixelFormat(PixelFormat format);