FFmpeg EGL/vaapi: Disable HW accelerated texture conversion when failing

This change applies to hardware accelerated texture conversion using EGL
and VA-API. If a call to eglCreateImage() inside
VAAPITextureConverter::getTextures() fails with EGL_BAD_MATCH error,
this change makes sure we fall back to software decoding by setting
this->rhi = nullptr.

This specific error can be reproduced in a Linux environment with both
a NVIDIA and AMD gpu, when NVIDIA is used as the EGL vendor.

By initializing the hw context manually using vaInitialize(), it is
possible to detect this problem before trying to decode frames. But we
can't since we let FFmpeg do this for us when we call
av_hwdevice_ctx_create().

The related qWarning() messages are also elaborated with error codes
and more relevant information.

Fixes: QTBUG-112312
Pick-to: 6.7 6.6 6.5
Change-Id: Ie90623c7b2b18b70b1000ae3ad6f96872933e884
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>

1 files changed, 16 insertions, 5 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp
index a7df486a2..88d6d4f0e 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp
@@ -301,17 +301,28 @@ TextureSet *VAAPITextureConverter::getTextures(AVFrame *frame)
         };
         images[i] = eglCreateImage(eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, img_attr);
         if (!images[i]) {
-            qWarning() << "eglCreateImage failed for plane" << i << Qt::hex << eglGetError();
-            return nullptr;
+            const GLenum error = eglGetError();
+            if (error == EGL_BAD_MATCH) {
+                qWarning() << "eglCreateImage failed for plane" << i << "with error code EGL_BAD_MATCH, "
+                              "disabling hardware acceleration. This could indicate an EGL implementation issue."
+                              "\nVAAPI driver: " << vaQueryVendorString(vaDisplay)
+                           << "\nEGL vendor:" << eglQueryString(eglDisplay, EGL_VENDOR);
+                this->rhi = nullptr; // Disabling texture conversion here to fix QTBUG-112312
+                return nullptr;
+            }
+            if (error) {
+                qWarning() << "eglCreateImage failed for plane" << i << "with error code" << error;
+                return nullptr;
+            }
         }
         functions.glActiveTexture(GL_TEXTURE0 + i);
         functions.glBindTexture(GL_TEXTURE_2D, glTextures[i]);
 
         PFNGLEGLIMAGETARGETTEXTURE2DOESPROC eglImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)this->eglImageTargetTexture2D;
         eglImageTargetTexture2D(GL_TEXTURE_2D, images[i]);
-        if (glGetError()) {
-            qWarning() << "eglImageTargetTexture2D failed";
-        }
+        GLenum error = glGetError();
+        if (error)
+            qWarning() << "eglImageTargetTexture2D failed with error code" << error;
     }
 
     for (int i = 0;  i < (int)prime.num_objects;  ++i)