Implement YUV 4:2:2 8bit support for QtQuick

This patch implements support for rendering of UYVY and YUYV sources when using
QtQuick for preview.

[ChangeLog][QtQuick][Rendering] Support for YUV 4:2:2 8bit has
been implemented for the QtQuick viewfinder.

Change-Id: I4d98f3c44240ee53f7708bc6bd84e7fb83aac069
Reviewed-by: Yoann Lopes <yoann.lopes@qt.io>

7 files changed, 147 insertions, 8 deletions

diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
index 599b70021..8080f3ec2 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
@@ -78,7 +78,7 @@ public:
     QSGVideoMaterialShader_Texture()
         : QSGMaterialShader()
     {
-        setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.vert"));
+        setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/monoplanarvideo.vert"));
         setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag"));
     }
 
diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
index 314a26264..68e6b456b 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
@@ -53,7 +53,8 @@ QList<QVideoFrame::PixelFormat> QSGVideoNodeFactory_YUV::supportedPixelFormats(
 
     if (handleType == QAbstractVideoBuffer::NoHandle) {
         formats << QVideoFrame::Format_YUV420P << QVideoFrame::Format_YV12
-                << QVideoFrame::Format_NV12 << QVideoFrame::Format_NV21;
+                << QVideoFrame::Format_NV12 << QVideoFrame::Format_NV21
+                << QVideoFrame::Format_UYVY << QVideoFrame::Format_YUYV;
     }
 
     return formats;
@@ -109,6 +110,55 @@ protected:
     int m_id_opacity;
 };
 
+class QSGVideoMaterialShader_UYVY : public QSGMaterialShader
+{
+public:
+    QSGVideoMaterialShader_UYVY()
+        : QSGMaterialShader()
+    {
+        setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/monoplanarvideo.vert"));
+        setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/uyvyvideo.frag"));
+    }
+
+    void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) Q_DECL_OVERRIDE;
+
+    char const *const *attributeNames() const Q_DECL_OVERRIDE {
+        static const char *names[] = {
+            "qt_VertexPosition",
+            "qt_VertexTexCoord",
+            0
+        };
+        return names;
+    }
+
+protected:
+    void initialize() Q_DECL_OVERRIDE {
+        m_id_matrix = program()->uniformLocation("qt_Matrix");
+        m_id_yuvtexture = program()->uniformLocation("yuvTexture");
+        m_id_imageWidth = program()->uniformLocation("imageWidth");
+        m_id_colorMatrix = program()->uniformLocation("colorMatrix");
+        m_id_opacity = program()->uniformLocation("opacity");
+        QSGMaterialShader::initialize();
+    }
+
+    int m_id_matrix;
+    int m_id_yuvtexture;
+    int m_id_imageWidth;
+    int m_id_colorMatrix;
+    int m_id_opacity;
+};
+
+
+class QSGVideoMaterialShader_YUYV : public QSGVideoMaterialShader_UYVY
+{
+public:
+    QSGVideoMaterialShader_YUYV()
+        : QSGVideoMaterialShader_UYVY()
+    {
+        setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/yuyvvideo.frag"));
+    }
+};
+
 
 class QSGVideoMaterialShader_YUV_BiPlanar_swizzle : public QSGVideoMaterialShader_YUV_BiPlanar
 {
@@ -152,13 +202,17 @@ public:
     ~QSGVideoMaterial_YUV();
 
     virtual QSGMaterialType *type() const {
-        static QSGMaterialType biPlanarType, biPlanarSwizzleType, triPlanarType;
+        static QSGMaterialType biPlanarType, biPlanarSwizzleType, triPlanarType, uyvyType, yuyvType;
 
         switch (m_format.pixelFormat()) {
         case QVideoFrame::Format_NV12:
             return &biPlanarType;
         case QVideoFrame::Format_NV21:
             return &biPlanarSwizzleType;
+        case QVideoFrame::Format_UYVY:
+            return &uyvyType;
+        case QVideoFrame::Format_YUYV:
+            return &yuyvType;
         default: // Currently: YUV420P and YV12
             return &triPlanarType;
         }
@@ -170,6 +224,10 @@ public:
             return new QSGVideoMaterialShader_YUV_BiPlanar;
         case QVideoFrame::Format_NV21:
             return new QSGVideoMaterialShader_YUV_BiPlanar_swizzle;
+        case QVideoFrame::Format_UYVY:
+            return new QSGVideoMaterialShader_UYVY;
+        case QVideoFrame::Format_YUYV:
+            return new QSGVideoMaterialShader_YUYV;
         default: // Currently: YUV420P and YV12
             return new QSGVideoMaterialShader_YUV_TriPlanar;
         }
@@ -230,6 +288,8 @@ QSGVideoMaterial_YUV::QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format) :
     case QVideoFrame::Format_YV12:
         m_planeCount = 3;
         break;
+    case QVideoFrame::Format_UYVY:
+    case QVideoFrame::Format_YUYV:
     default:
         m_planeCount = 1;
         break;
@@ -293,7 +353,15 @@ void QSGVideoMaterial_YUV::bind()
             functions->glGetIntegerv(GL_UNPACK_ALIGNMENT, &previousAlignment);
             functions->glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
 
-            if (m_format.pixelFormat() == QVideoFrame::Format_NV12
+             if (m_format.pixelFormat() == QVideoFrame::Format_UYVY
+              || m_format.pixelFormat() == QVideoFrame::Format_YUYV) {
+                int fw = m_frame.width() / 2;
+                m_planeWidth[0] = fw;
+
+                functions->glActiveTexture(GL_TEXTURE0);
+                bindTexture(m_textureIds[0], fw, m_frame.height(), m_frame.bits(), GL_RGBA);
+
+            } else if (m_format.pixelFormat() == QVideoFrame::Format_NV12
                     || m_format.pixelFormat() == QVideoFrame::Format_NV21) {
                 const int y = 0;
                 const int uv = 1;
@@ -338,7 +406,6 @@ void QSGVideoMaterial_YUV::bind()
 void QSGVideoMaterial_YUV::bindTexture(int id, int w, int h, const uchar *bits, GLenum format)
 {
     QOpenGLFunctions *functions = QOpenGLContext::currentContext()->functions();
-
     functions->glBindTexture(GL_TEXTURE_2D, id);
     functions->glTexImage2D(GL_TEXTURE_2D, 0, format, w, h, 0, format, GL_UNSIGNED_BYTE, bits);
     functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
@@ -399,4 +466,28 @@ void QSGVideoMaterialShader_YUV_TriPlanar::updateState(const RenderState &state,
     program()->setUniformValue(m_id_plane3Width, mat->m_planeWidth[2]);
 }
 
+void QSGVideoMaterialShader_UYVY::updateState(const RenderState &state,
+                                                       QSGMaterial *newMaterial,
+                                                       QSGMaterial *oldMaterial)
+{
+    Q_UNUSED(oldMaterial);
+
+    QSGVideoMaterial_YUV *mat = static_cast<QSGVideoMaterial_YUV *>(newMaterial);
+
+    program()->setUniformValue(m_id_yuvtexture, 0);
+
+    mat->bind();
+
+    program()->setUniformValue(m_id_colorMatrix, mat->m_colorMatrix);
+    program()->setUniformValue(m_id_imageWidth, mat->m_frame.width());
+
+    if (state.isOpacityDirty()) {
+        mat->m_opacity = state.opacity();
+        program()->setUniformValue(m_id_opacity, GLfloat(mat->m_opacity));
+    }
+
+    if (state.isMatrixDirty())
+        program()->setUniformValue(m_id_matrix, state.combinedMatrix());
+}
+
 QT_END_NAMESPACE
diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro
index e1425c3ec..e4e157a54 100644
--- a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro
+++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro
@@ -35,7 +35,7 @@ RESOURCES += \
     qtmultimediaquicktools.qrc
 
 OTHER_FILES += \
-    shaders/rgbvideo.vert \
+    shaders/monoplanarvideo.vert \
     shaders/rgbvideo_padded.vert \
     shaders/rgbvideo.frag \
     shaders/rgbvideo_swizzle.frag \
@@ -43,6 +43,8 @@ OTHER_FILES += \
     shaders/biplanaryuvvideo.frag \
     shaders/biplanaryuvvideo_swizzle.frag \
     shaders/triplanaryuvvideo.vert \
-    shaders/triplanaryuvvideo.frag
+    shaders/triplanaryuvvideo.frag \
+    shaders/uyvyvideo.frag \
+    shaders/yuyvvideo.frag
 
 load(qt_module)
diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc
index b0a7f0789..d5efb1f3e 100644
--- a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc
+++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc
@@ -1,6 +1,6 @@
 <RCC>
     <qresource prefix="/qtmultimediaquicktools">
-        <file>shaders/rgbvideo.vert</file>
+        <file>shaders/monoplanarvideo.vert</file>
         <file>shaders/rgbvideo.frag</file>
         <file>shaders/rgbvideo_swizzle.frag</file>
         <file>shaders/rgbvideo_padded.vert</file>
@@ -9,5 +9,7 @@
         <file>shaders/biplanaryuvvideo_swizzle.frag</file>
         <file>shaders/triplanaryuvvideo.frag</file>
         <file>shaders/triplanaryuvvideo.vert</file>
+        <file>shaders/uyvyvideo.frag</file>
+        <file>shaders/yuyvvideo.frag</file>
     </qresource>
 </RCC>
diff --git a/src/qtmultimediaquicktools/shaders/rgbvideo.vert b/src/qtmultimediaquicktools/shaders/monoplanarvideo.vert
index 19915f2ff..19915f2ff 100644
--- a/src/qtmultimediaquicktools/shaders/rgbvideo.vert
+++ b/src/qtmultimediaquicktools/shaders/monoplanarvideo.vert
diff --git a/src/qtmultimediaquicktools/shaders/uyvyvideo.frag b/src/qtmultimediaquicktools/shaders/uyvyvideo.frag
new file mode 100644
index 000000000..8e34f7d33
--- /dev/null
+++ b/src/qtmultimediaquicktools/shaders/uyvyvideo.frag
@@ -0,0 +1,22 @@
+uniform sampler2D yuvTexture; // UYVY macropixel texture passed as RGBA format
+uniform float imageWidth; // The UYVY texture appears to the shader with 1/2 the image width since we use the RGBA format to pass UYVY
+uniform mediump mat4 colorMatrix;
+uniform lowp float opacity;
+
+varying highp vec2 qt_TexCoord;
+
+void main()
+{
+   // For U0 Y0 V0 Y1 macropixel, lookup Y0 or Y1 based on whether
+   // the original texture x coord is even or odd.
+   mediump float Y;
+   if (fract(floor(qt_TexCoord.x * imageWidth + 0.5) / 2.0) > 0.0)
+       Y = texture2D(yuvTexture, qt_TexCoord).a;       // odd so choose Y1
+   else
+       Y = texture2D(yuvTexture, qt_TexCoord).g;       // even so choose Y0
+   mediump float Cb = texture2D(yuvTexture, qt_TexCoord).r;
+   mediump float Cr = texture2D(yuvTexture, qt_TexCoord).b;
+
+   mediump vec4 color = vec4(Y, Cb, Cr, 1.0);
+   gl_FragColor = colorMatrix * color * opacity;
+}
diff --git a/src/qtmultimediaquicktools/shaders/yuyvvideo.frag b/src/qtmultimediaquicktools/shaders/yuyvvideo.frag
new file mode 100644
index 000000000..876771008
--- /dev/null
+++ b/src/qtmultimediaquicktools/shaders/yuyvvideo.frag
@@ -0,0 +1,22 @@
+uniform sampler2D yuvTexture; // YUYV macropixel texture passed as RGBA format
+uniform float imageWidth; // The YUYV texture appears to the shader with 1/2 the image width since we use the RGBA format to pass YUYV
+uniform mediump mat4 colorMatrix;
+uniform lowp float opacity;
+
+varying highp vec2 qt_TexCoord;
+
+void main()
+{
+   // For Y0 U0 Y1 V0  macropixel, lookup Y0 or Y1 based on whether
+   // the original texture x coord is even or odd.
+   mediump float Y;
+   if (fract(floor(qt_TexCoord.x * imageWidth + 0.5) / 2.0) > 0.0)
+       Y = texture2D(yuvTexture, qt_TexCoord).b;       // odd so choose Y1
+   else
+       Y = texture2D(yuvTexture, qt_TexCoord).r;       // even so choose Y0
+   mediump float Cb = texture2D(yuvTexture, qt_TexCoord).g;
+   mediump float Cr = texture2D(yuvTexture, qt_TexCoord).a;
+
+   mediump vec4 color = vec4(Y, Cb, Cr, 1.0);
+   gl_FragColor = colorMatrix * color * opacity;
+}