More work to enable HW decoding for Qt Quick

Request GL textures from the pipeline for now. Some initial code
to also use the texture upload meta functionality in gstreamer,
but that will require some more work so we don't make the GL
context current in the wrong thread.

The gstreamer VAAPI elements on AMD hardware (or in general...)
seem to have some bugs. Converting a VASurface to a GL texture
using the texture upload meta doesn't create an ARGB texture as
promised, but does write some YUV data into the texture. And
trying to map a SW buffer received from the VAAPI decoders fails.

Change-Id: I9b629eb84f3f32adc23ae2e2fd1cd3e42e6afbc0
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

1 files changed, 18 insertions, 10 deletions

diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
index efefaf0f1..182e98610 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
@@ -354,12 +354,12 @@ void QSGVideoMaterialRhiShader_NV12::mapFrame(QSGVideoMaterial_YUV *m)
 
     if (m->m_frame.handleType() == QVideoFrame::RhiTextureHandle) {
         m->m_planeWidth[0] = m->m_planeWidth[1] = 1;
-        auto textures = m->m_frame.handle().toList();
-        if (!textures.isEmpty()) {
+        quint64 textures[2] = { m->m_frame.textureHandle(0), m->m_frame.textureHandle(1) };
+        if (textures[0] && textures[1]) {
             auto w = m->m_frame.size().width();
             auto h = m->m_frame.size().height();
-            m->m_textures[0]->setNativeObject(textures[0].toULongLong(), {w, h}, QRhiTexture::R8);
-            m->m_textures[1]->setNativeObject(textures[1].toULongLong(), {w / 2, h / 2}, QRhiTexture::RG8);
+            m->m_textures[0]->setNativeObject(textures[0], {w, h}, QRhiTexture::R8);
+            m->m_textures[1]->setNativeObject(textures[1], {w / 2, h / 2}, QRhiTexture::RG8);
         } else {
             qWarning() << "NV12/NV21 requires 2 textures";
         }
@@ -367,8 +367,12 @@ void QSGVideoMaterialRhiShader_NV12::mapFrame(QSGVideoMaterial_YUV *m)
         return;
     }
 
-    if (!m->m_frame.map(QVideoFrame::ReadOnly))
+    if (!m->m_frame.map(QVideoFrame::ReadOnly)) {
+        qWarning()<< "NV12: Couldn't map frame";
+        m->m_textures[0]->setData(QRhiTexture::RG8, QSize(1, 1), (const uchar *)"\0\0", 2);
+        m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(1, 1), (const uchar *)"\0\0\0\0", 4);
         return;
+    }
 
     int y = 0;
     int uv = 1;
@@ -392,12 +396,12 @@ void QSGVideoMaterialRhiShader_P010::mapFrame(QSGVideoMaterial_YUV *m)
 
     if (m->m_frame.handleType() == QVideoFrame::RhiTextureHandle) {
         m->m_planeWidth[0] = m->m_planeWidth[1] = 1;
-        auto textures = m->m_frame.handle().toList();
-        if (!textures.isEmpty()) {
+        quint64 textures[2] = { m->m_frame.textureHandle(0), m->m_frame.textureHandle(1) };
+        if (textures[0] && textures[1]) {
             auto w = m->m_frame.size().width();
             auto h = m->m_frame.size().height();
-            m->m_textures[0]->setNativeObject(textures[0].toULongLong(), {w, h}, QRhiTexture::RG8);
-            m->m_textures[1]->setNativeObject(textures[1].toULongLong(), {w / 2, h / 2}, QRhiTexture::BGRA8);
+            m->m_textures[0]->setNativeObject(textures[0], {w, h}, QRhiTexture::RG8);
+            m->m_textures[1]->setNativeObject(textures[1], {w / 2, h / 2}, QRhiTexture::BGRA8);
         } else {
             qWarning() << "P010/P016 requires 2 textures";
         }
@@ -405,8 +409,12 @@ void QSGVideoMaterialRhiShader_P010::mapFrame(QSGVideoMaterial_YUV *m)
         return;
     }
 
-    if (!m->m_frame.map(QVideoFrame::ReadOnly))
+    if (!m->m_frame.map(QVideoFrame::ReadOnly)) {
+        qWarning()<< "NV12: Couldn't map frame";
+        m->m_textures[0]->setData(QRhiTexture::RG8, QSize(1, 1), (const uchar *)"\0\0", 2);
+        m->m_textures[1]->setData(QRhiTexture::BGRA8, QSize(1, 1), (const uchar *)"\0\0\0\0", 4);
         return;
+    }
 
     int y = 0;
     int uv = 1;