Various fixes to the shader code

Tested all formats that could be tested through gstreamer and
ensured our shaders to the right thing.

Required adding a width uniform, so we can pick the correct
Y value for YUYV non planar formats.

The IMC formats should work, but are untested as gstreamer
does not support them.

Change-Id: Ib0fe0a7c7d97eb81a75f636c752a85f51a327183
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

17 files changed, 127 insertions, 34 deletions

diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt
index e444d0fa3..0de527d4b 100644
--- a/src/multimedia/CMakeLists.txt
+++ b/src/multimedia/CMakeLists.txt
@@ -483,8 +483,11 @@ qt_internal_add_shaders(Multimedia "shaders"
         "shaders/y.frag"
         "shaders/nv12.frag"
         "shaders/nv21.frag"
+        "shaders/imc2.frag"
+        "shaders/imc4.frag"
         "shaders/uyvy.frag"
-        "shaders/yuv_yv.frag"
+        "shaders/yuv_triplanar.frag"
+        "shaders/yvu_triplanar.frag"
         "shaders/yuyv.frag"
         "shaders/ayuv.frag"
 )
diff --git a/src/multimedia/platform/gstreamer/common/qgstvideorenderersink.cpp b/src/multimedia/platform/gstreamer/common/qgstvideorenderersink.cpp
index ff8daa24b..49bf61a62 100644
--- a/src/multimedia/platform/gstreamer/common/qgstvideorenderersink.cpp
+++ b/src/multimedia/platform/gstreamer/common/qgstvideorenderersink.cpp
@@ -106,16 +106,13 @@ QGstMutableCaps QGstVideoRenderer::getCaps()
 //                       << QVideoSurfaceFormat::Format_NV21
                        << QVideoSurfaceFormat::Format_AYUV444
 //                       << QVideoSurfaceFormat::Format_P010
-//                       << QVideoSurfaceFormat::Format_Y8
                        << QVideoSurfaceFormat::Format_RGB32
                        << QVideoSurfaceFormat::Format_BGR32
                        << QVideoSurfaceFormat::Format_ARGB32
                        << QVideoSurfaceFormat::Format_ABGR32
                        << QVideoSurfaceFormat::Format_BGRA32
-//                       << QVideoSurfaceFormat::Format_RGB555
-//                       << QVideoSurfaceFormat::Format_BGR555
+//                       << QVideoSurfaceFormat::Format_Y8
 //                       << QVideoSurfaceFormat::Format_Y16
-//                       << QVideoSurfaceFormat::Format_RGB565
             ;
         // Even if the surface does not support gl textures,
         // glupload will be added to the pipeline and GLMemory will be requested.
@@ -137,16 +134,17 @@ QGstMutableCaps QGstVideoRenderer::getCaps()
                    << QVideoSurfaceFormat::Format_NV21
                    << QVideoSurfaceFormat::Format_AYUV444
                    << QVideoSurfaceFormat::Format_P010
-                   << QVideoSurfaceFormat::Format_Y8
                    << QVideoSurfaceFormat::Format_RGB32
                    << QVideoSurfaceFormat::Format_BGR32
                    << QVideoSurfaceFormat::Format_ARGB32
                    << QVideoSurfaceFormat::Format_ABGR32
                    << QVideoSurfaceFormat::Format_BGRA32
-                   << QVideoSurfaceFormat::Format_RGB555
-                   << QVideoSurfaceFormat::Format_BGR555
+                   << QVideoSurfaceFormat::Format_Y8
                    << QVideoSurfaceFormat::Format_Y16
-                   << QVideoSurfaceFormat::Format_RGB565;
+//                   << QVideoSurfaceFormat::Format_RGB555
+//                   << QVideoSurfaceFormat::Format_BGR555
+//                   << QVideoSurfaceFormat::Format_RGB565
+        ;
     caps.addPixelFormats(formats);
     qDebug() << "CAPS:" << caps.toString();
     return caps;
diff --git a/src/multimedia/shaders/abgr.frag b/src/multimedia/shaders/abgr.frag
index e862d3319..2b2812a33 100644
--- a/src/multimedia/shaders/abgr.frag
+++ b/src/multimedia/shaders/abgr.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D rgbTexture;
diff --git a/src/multimedia/shaders/argb.frag b/src/multimedia/shaders/argb.frag
index 732320c91..6b9abade7 100644
--- a/src/multimedia/shaders/argb.frag
+++ b/src/multimedia/shaders/argb.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D rgbTexture;
diff --git a/src/multimedia/shaders/ayuv.frag b/src/multimedia/shaders/ayuv.frag
index 6ae19b166..51427eed0 100644
--- a/src/multimedia/shaders/ayuv.frag
+++ b/src/multimedia/shaders/ayuv.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
diff --git a/src/multimedia/shaders/bgra.frag b/src/multimedia/shaders/bgra.frag
index 6c66e6bd3..938dfbd72 100644
--- a/src/multimedia/shaders/bgra.frag
+++ b/src/multimedia/shaders/bgra.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D rgbTexture;
diff --git a/src/multimedia/shaders/imc2.frag b/src/multimedia/shaders/imc2.frag
new file mode 100644
index 000000000..66ed4fe17
--- /dev/null
+++ b/src/multimedia/shaders/imc2.frag
@@ -0,0 +1,24 @@
+#version 440
+
+layout(location = 0) in vec2 texCoord;
+layout(location = 0) out vec4 fragColor;
+
+layout(std140, binding = 0) uniform buf {
+    mat4 matrix;
+    mat4 colorMatrix;
+    float opacity;
+    float width;
+} ubuf;
+
+layout(binding = 1) uniform sampler2D plane1Texture;
+layout(binding = 2) uniform sampler2D plane2Texture;
+
+void main()
+{
+    float Y = texture(plane1Texture, texCoord).r;
+    float x = texCoord.x/2.;
+    float U = texture(plane2Texture, vec2(x, texCoord.y)).r;
+    float V = texture(plane2Texture, vec2(x + .5, texCoord.y)).r;
+    vec4 color = vec4(Y, U, V, 1.);
+    fragColor = ubuf.colorMatrix * color * ubuf.opacity;
+}
diff --git a/src/multimedia/shaders/imc4.frag b/src/multimedia/shaders/imc4.frag
new file mode 100644
index 000000000..66ed4fe17
--- /dev/null
+++ b/src/multimedia/shaders/imc4.frag
@@ -0,0 +1,24 @@
+#version 440
+
+layout(location = 0) in vec2 texCoord;
+layout(location = 0) out vec4 fragColor;
+
+layout(std140, binding = 0) uniform buf {
+    mat4 matrix;
+    mat4 colorMatrix;
+    float opacity;
+    float width;
+} ubuf;
+
+layout(binding = 1) uniform sampler2D plane1Texture;
+layout(binding = 2) uniform sampler2D plane2Texture;
+
+void main()
+{
+    float Y = texture(plane1Texture, texCoord).r;
+    float x = texCoord.x/2.;
+    float U = texture(plane2Texture, vec2(x, texCoord.y)).r;
+    float V = texture(plane2Texture, vec2(x + .5, texCoord.y)).r;
+    vec4 color = vec4(Y, U, V, 1.);
+    fragColor = ubuf.colorMatrix * color * ubuf.opacity;
+}
diff --git a/src/multimedia/shaders/nv12.frag b/src/multimedia/shaders/nv12.frag
index 49c333c98..277f9dc0b 100644
--- a/src/multimedia/shaders/nv12.frag
+++ b/src/multimedia/shaders/nv12.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
diff --git a/src/multimedia/shaders/nv21.frag b/src/multimedia/shaders/nv21.frag
index 5876c504a..29890c7c0 100644
--- a/src/multimedia/shaders/nv21.frag
+++ b/src/multimedia/shaders/nv21.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
diff --git a/src/multimedia/shaders/uyvy.frag b/src/multimedia/shaders/uyvy.frag
index d48e9a5b5..8132407ca 100644
--- a/src/multimedia/shaders/uyvy.frag
+++ b/src/multimedia/shaders/uyvy.frag
@@ -7,13 +7,16 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
-layout(binding = 2) uniform sampler2D plane2Texture;
 
 void main()
 {
-    vec3 YUV = vec3(texture(plane1Texture, texCoord).g, texture(plane2Texture, texCoord).rb);
-    fragColor = ubuf.colorMatrix * vec4(YUV, 1.0) * ubuf.opacity;
+    float x = texCoord.x * ubuf.width;
+    bool rightSubPixel = bool(int(x) & 1);
+    float Y = rightSubPixel ? texture(plane1Texture, texCoord).a : texture(plane1Texture, texCoord).g;
+    vec2 UV = texture(plane1Texture, texCoord).br;
+    fragColor = ubuf.colorMatrix * vec4(Y, UV, 1.0) * ubuf.opacity;
 }
diff --git a/src/multimedia/shaders/vertex.vert b/src/multimedia/shaders/vertex.vert
index 9cddf2a39..068a4157c 100644
--- a/src/multimedia/shaders/vertex.vert
+++ b/src/multimedia/shaders/vertex.vert
@@ -9,6 +9,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 out gl_PerVertex { vec4 gl_Position; };
diff --git a/src/multimedia/shaders/y.frag b/src/multimedia/shaders/y.frag
index f32bff891..82c75ec78 100644
--- a/src/multimedia/shaders/y.frag
+++ b/src/multimedia/shaders/y.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
diff --git a/src/multimedia/shaders/yuv_yv.frag b/src/multimedia/shaders/yuv_triplanar.frag
index 26fa202f5..239c1dcfd 100644
--- a/src/multimedia/shaders/yuv_yv.frag
+++ b/src/multimedia/shaders/yuv_triplanar.frag
@@ -7,6 +7,7 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
diff --git a/src/multimedia/shaders/yuyv.frag b/src/multimedia/shaders/yuyv.frag
index 6838596e8..c4dd5e6ff 100644
--- a/src/multimedia/shaders/yuyv.frag
+++ b/src/multimedia/shaders/yuyv.frag
@@ -7,13 +7,16 @@ layout(std140, binding = 0) uniform buf {
     mat4 matrix;
     mat4 colorMatrix;
     float opacity;
+    float width;
 } ubuf;
 
 layout(binding = 1) uniform sampler2D plane1Texture;
-layout(binding = 2) uniform sampler2D plane2Texture;
 
 void main()
 {
-    vec3 YUV = vec3(texture(plane1Texture, texCoord).r, texture(plane2Texture, texCoord).ga);
-    fragColor = ubuf.colorMatrix * vec4(YUV, 1.0) * ubuf.opacity;
+    float x = texCoord.x * ubuf.width;
+    bool rightSubPixel = bool(int(x) & 1);
+    float Y = rightSubPixel ? texture(plane1Texture, texCoord).r : texture(plane1Texture, texCoord).b;
+    vec2 UV = texture(plane1Texture, texCoord).ga;
+    fragColor = ubuf.colorMatrix * vec4(Y, UV, 1.0) * ubuf.opacity;
 }
diff --git a/src/multimedia/shaders/yvu_triplanar.frag b/src/multimedia/shaders/yvu_triplanar.frag
new file mode 100644
index 000000000..edd9947e3
--- /dev/null
+++ b/src/multimedia/shaders/yvu_triplanar.frag
@@ -0,0 +1,24 @@
+#version 440
+
+layout(location = 0) in vec2 texCoord;
+layout(location = 0) out vec4 fragColor;
+
+layout(std140, binding = 0) uniform buf {
+    mat4 matrix;
+    mat4 colorMatrix;
+    float opacity;
+    float width;
+} ubuf;
+
+layout(binding = 1) uniform sampler2D plane1Texture;
+layout(binding = 2) uniform sampler2D plane2Texture;
+layout(binding = 3) uniform sampler2D plane3Texture;
+
+void main()
+{
+    float Y = texture(plane1Texture, texCoord).r;
+    float V = texture(plane2Texture, texCoord).r;
+    float U = texture(plane3Texture, texCoord).r;
+    vec4 color = vec4(Y, U, V, 1.);
+    fragColor = ubuf.colorMatrix * color * ubuf.opacity;
+}
diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp
index 5ccd82c76..2d7832020 100644
--- a/src/multimedia/video/qvideotexturehelper.cpp
+++ b/src/multimedia/video/qvideotexturehelper.cpp
@@ -133,42 +133,42 @@ static const TextureDescription descriptions[QVideoSurfaceFormat::NPixelFormats]
     // Format_UYVY
     { 1,
      { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
-     { { 1, 1 }, { 1, 1 }, { 1, 1 } }
+     { { 2, 1 }, { 1, 1 }, { 1, 1 } }
     },
     // Format_YUYV
     { 1,
      { QRhiTexture::BGRA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
-     { { 1, 1 }, { 1, 1 }, { 1, 1 } }
+     { { 2, 1 }, { 1, 1 }, { 1, 1 } }
     },
     // Format_NV12
     { 2,
-     { QRhiTexture::RG8, QRhiTexture::R8, QRhiTexture::UnknownFormat },
+     { QRhiTexture::R8, QRhiTexture::RG8, QRhiTexture::UnknownFormat },
      { { 1, 1 }, { 2, 2 }, { 1, 1 } }
     },
     // Format_NV21
     { 2,
-     { QRhiTexture::RG8, QRhiTexture::R8, QRhiTexture::UnknownFormat },
+     { QRhiTexture::R8, QRhiTexture::RG8, QRhiTexture::UnknownFormat },
      { { 1, 1 }, { 2, 2 }, { 1, 1 } }
     },
     // Format_IMC1
     { 3,
      { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
-     { { 1, 1 }, { 1, 1 }, { 1, 1 } }
+     { { 1, 1 }, { 2, 2 }, { 2, 2 } }
     },
     // Format_IMC2
     { 2,
      { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat },
-     { { 1, 1 }, { 1, 1 }, { 1, 1 } }
+     { { 1, 1 }, { 1, 2 }, { 1, 1 } }
     },
     // Format_IMC3
     { 3,
      { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
-     { { 1, 1 }, { 1, 1 }, { 1, 1 } }
+     { { 1, 1 }, { 2, 2 }, { 2, 2 } }
     },
     // Format_IMC4
     { 2,
      { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat },
-     { { 1, 1 }, { 1, 1 }, { 1, 1 } }
+     { { 1, 1 }, { 1, 2 }, { 1, 1 } }
     },
     // Format_Y8
     { 1,
@@ -221,10 +221,6 @@ QString fragmentShaderFileName(QVideoSurfaceFormat::PixelFormat format)
     case QVideoSurfaceFormat::Format_BGR565:
     case QVideoSurfaceFormat::Format_BGR555:
 
-    case QVideoSurfaceFormat::Format_IMC1:
-    case QVideoSurfaceFormat::Format_IMC2:
-    case QVideoSurfaceFormat::Format_IMC3:
-    case QVideoSurfaceFormat::Format_IMC4:
         return QString();
 
     case QVideoSurfaceFormat::Format_Y8:
@@ -245,8 +241,15 @@ QString fragmentShaderFileName(QVideoSurfaceFormat::PixelFormat format)
         return QStringLiteral(":/qt-project.org/multimedia/shaders/abgr.frag.qsb");
     case QVideoSurfaceFormat::Format_YUV420P:
     case QVideoSurfaceFormat::Format_YUV422P:
+    case QVideoSurfaceFormat::Format_IMC3:
+        return QStringLiteral(":/qt-project.org/multimedia/shaders/yuv_triplanar.frag.qsb");
     case QVideoSurfaceFormat::Format_YV12:
-        return QStringLiteral(":/qt-project.org/multimedia/shaders/yuv_yv.frag.qsb");
+    case QVideoSurfaceFormat::Format_IMC1:
+        return QStringLiteral(":/qt-project.org/multimedia/shaders/yvu_triplanar.frag.qsb");
+    case QVideoSurfaceFormat::Format_IMC2:
+        return QStringLiteral(":/qt-project.org/multimedia/shaders/imc2.frag.qsb");
+    case QVideoSurfaceFormat::Format_IMC4:
+        return QStringLiteral(":/qt-project.org/multimedia/shaders/imc4.frag.qsb");
     case QVideoSurfaceFormat::Format_UYVY:
         return QStringLiteral(":/qt-project.org/multimedia/shaders/uyvy.frag.qsb");
     case QVideoSurfaceFormat::Format_YUYV:
@@ -298,12 +301,8 @@ QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &tran
     case QVideoSurfaceFormat::Format_RGB555:
     case QVideoSurfaceFormat::Format_BGR565:
     case QVideoSurfaceFormat::Format_BGR555:
-
-    case QVideoSurfaceFormat::Format_IMC1:
-    case QVideoSurfaceFormat::Format_IMC2:
-    case QVideoSurfaceFormat::Format_IMC3:
-    case QVideoSurfaceFormat::Format_IMC4:
         return QByteArray();
+
     case QVideoSurfaceFormat::Format_ARGB32:
     case QVideoSurfaceFormat::Format_ARGB32_Premultiplied:
     case QVideoSurfaceFormat::Format_RGB32:
@@ -315,6 +314,10 @@ QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &tran
     case QVideoSurfaceFormat::Format_Y8:
     case QVideoSurfaceFormat::Format_Y16:
         break;
+    case QVideoSurfaceFormat::Format_IMC1:
+    case QVideoSurfaceFormat::Format_IMC2:
+    case QVideoSurfaceFormat::Format_IMC3:
+    case QVideoSurfaceFormat::Format_IMC4:
     case QVideoSurfaceFormat::Format_AYUV444:
     case QVideoSurfaceFormat::Format_AYUV444_Premultiplied:
     case QVideoSurfaceFormat::Format_YUV420P:
@@ -330,11 +333,13 @@ QByteArray uniformData(const QVideoSurfaceFormat &format, const QMatrix4x4 &tran
         break;
     }
     // { matrix4x4, colorMatrix, opacity, planeWidth[3] }
-    QByteArray buf(64*2 + 4, Qt::Uninitialized);
+    QByteArray buf(64*2 + 4 + 4, Qt::Uninitialized);
     char *data = buf.data();
     memcpy(data, transform.constData(), 64);
     memcpy(data + 64, cmat.constData(), 64);
     memcpy(data + 64 + 64, &opacity, 4);
+    float width = format.frameWidth();
+    memcpy(data + 64 + 64 + 4, &width, 4);
     return buf;
 }