linux-dmabuf-unstable-v1: Fix importing dmabuf buffers with modifiers

Depending on the used modifiers, there are cases where dmabuf buffers
may be transported as multiple dmabuf planes, although they are
conceptually single plane buffers. An example of a modifier that works
like this is I915_FORMAT_MOD_Y_TILED_CCS. In such cases we need to
create a single EGLImage by merging all the dmabuf planes.

Almost all of the currently supported DRM formats are conceptually
single plane, so they require such merging of the dmabuf planes. The
only formats that need to be mapped to multiple EGLImages are some YUV
based ones, which we are already importing in a different way
(see initYuvTexture).

In addition, ensure that we don't pass to EGL a modifier value of
DRM_FORMAT_MOD_INVALID, since this is not an acceptable value.
To indicate a lack of modifiers skip the relevant
EGL_DMA_BUF_*_MODIFIER_* attributes altogether.

Sponsored-by: LG Electronics Inc.
Change-Id: I32a717f3be20927f2f3f4aa0fc6c4f7a5250db33
Reviewed-by: Eskil Abrahamsen Blomfeldt <eskil.abrahamsen-blomfeldt@qt.io>
(cherry picked from commit 8163327dcb3d37e9f3011bd9bec88848b2ce3b6c)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

1 files changed, 61 insertions, 68 deletions

diff --git a/src/hardwareintegration/compositor/linux-dmabuf-unstable-v1/linuxdmabufclientbufferintegration.cpp b/src/hardwareintegration/compositor/linux-dmabuf-unstable-v1/linuxdmabufclientbufferintegration.cpp
index d3690842e..90c104c28 100644
--- a/src/hardwareintegration/compositor/linux-dmabuf-unstable-v1/linuxdmabufclientbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/linux-dmabuf-unstable-v1/linuxdmabufclientbufferintegration.cpp
@@ -105,6 +105,11 @@ static QOpenGLTexture::TextureFormat openGLFormatFromBufferFormat(QWaylandBuffer
     }
 }
 
+// Initialize the EGLImage for a dmabuf buffer which conceptually consists of a
+// single plane. Note that depending on the modifiers, the buffer may be actually
+// transported as multiple dmabuf planes which must be combined into a single
+// EGLImage. For formats where the buffer needs to be represented as multiple
+// EGLImages (e.g., various YUV formats) a different approach is required.
 bool LinuxDmabufClientBufferIntegration::initSimpleTexture(LinuxDmabufWlBuffer *dmabufBuffer)
 {
     bool success = true;
@@ -118,79 +123,67 @@ bool LinuxDmabufClientBufferIntegration::initSimpleTexture(LinuxDmabufWlBuffer *
         success = false;
     }
 
-    for (uint32_t i = 0; i < dmabufBuffer->planesNumber(); ++i) {
-        QVarLengthArray<EGLint, 17> attribs;
-        switch (i) {
-        case 0:
-            attribs = {
-                EGL_WIDTH,                          dmabufBuffer->size().width(),
-                EGL_HEIGHT,                         dmabufBuffer->size().height(),
-                EGL_LINUX_DRM_FOURCC_EXT,           EGLint(dmabufBuffer->drmFormat()),
-                EGL_DMA_BUF_PLANE0_FD_EXT,          dmabufBuffer->plane(i).fd,
-                EGL_DMA_BUF_PLANE0_OFFSET_EXT,      EGLint(dmabufBuffer->plane(i).offset),
-                EGL_DMA_BUF_PLANE0_PITCH_EXT,       EGLint(dmabufBuffer->plane(i).stride),
-                EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, EGLint(dmabufBuffer->plane(i).modifiers & 0xffffffff),
-                EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, EGLint(dmabufBuffer->plane(i).modifiers >> 32),
-                EGL_NONE
-            };
-            break;
-        case 1:
-            attribs = {
-                EGL_WIDTH,                          dmabufBuffer->size().width(),
-                EGL_HEIGHT,                         dmabufBuffer->size().height(),
-                EGL_LINUX_DRM_FOURCC_EXT,           EGLint(dmabufBuffer->drmFormat()),
-                EGL_DMA_BUF_PLANE1_FD_EXT,          dmabufBuffer->plane(i).fd,
-                EGL_DMA_BUF_PLANE1_OFFSET_EXT,      EGLint(dmabufBuffer->plane(i).offset),
-                EGL_DMA_BUF_PLANE1_PITCH_EXT,       EGLint(dmabufBuffer->plane(i).stride),
-                EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT, EGLint(dmabufBuffer->plane(i).modifiers & 0xffffffff),
-                EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT, EGLint(dmabufBuffer->plane(i).modifiers >> 32),
-                EGL_NONE
-            };
-            break;
-        case 2:
-            attribs = {
-                EGL_WIDTH,                          dmabufBuffer->size().width(),
-                EGL_HEIGHT,                         dmabufBuffer->size().height(),
-                EGL_LINUX_DRM_FOURCC_EXT,           EGLint(dmabufBuffer->drmFormat()),
-                EGL_DMA_BUF_PLANE2_FD_EXT,          dmabufBuffer->plane(i).fd,
-                EGL_DMA_BUF_PLANE2_OFFSET_EXT,      EGLint(dmabufBuffer->plane(i).offset),
-                EGL_DMA_BUF_PLANE2_PITCH_EXT,       EGLint(dmabufBuffer->plane(i).stride),
-                EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT, EGLint(dmabufBuffer->plane(i).modifiers & 0xffffffff),
-                EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT, EGLint(dmabufBuffer->plane(i).modifiers >> 32),
-                EGL_NONE
-            };
-            break;
-        case 3:
-            attribs = {
-                EGL_WIDTH,                          dmabufBuffer->size().width(),
-                EGL_HEIGHT,                         dmabufBuffer->size().height(),
-                EGL_LINUX_DRM_FOURCC_EXT,           EGLint(dmabufBuffer->drmFormat()),
-                EGL_DMA_BUF_PLANE3_FD_EXT,          dmabufBuffer->plane(i).fd,
-                EGL_DMA_BUF_PLANE3_OFFSET_EXT,      EGLint(dmabufBuffer->plane(i).offset),
-                EGL_DMA_BUF_PLANE3_PITCH_EXT,       EGLint(dmabufBuffer->plane(i).stride),
-                EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT, EGLint(dmabufBuffer->plane(i).modifiers & 0xffffffff),
-                EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT, EGLint(dmabufBuffer->plane(i).modifiers >> 32),
-                EGL_NONE
-            };
+    // 6 entries for the common attribs plus 10 per possible plane, plus 1 for
+    // the final EGL_NONE sentinel.
+    QVarLengthArray<EGLint, 6 + 10 * 4 + 1> attribs;
+
+    attribs.append(EGL_WIDTH);
+    attribs.append(dmabufBuffer->size().width());
+    attribs.append(EGL_HEIGHT);
+    attribs.append(dmabufBuffer->size().height());
+    attribs.append(EGL_LINUX_DRM_FOURCC_EXT);
+    attribs.append(EGLint(dmabufBuffer->drmFormat()));
+
+#define ADD_PLANE_ATTRIBS(plane_idx) { \
+    attribs.append(EGL_DMA_BUF_PLANE ## plane_idx ## _FD_EXT); \
+    attribs.append(dmabufBuffer->plane(plane_idx).fd); \
+    attribs.append(EGL_DMA_BUF_PLANE ## plane_idx ## _OFFSET_EXT); \
+    attribs.append(EGLint(dmabufBuffer->plane(plane_idx).offset)); \
+    attribs.append(EGL_DMA_BUF_PLANE ## plane_idx ## _PITCH_EXT); \
+    attribs.append(EGLint(dmabufBuffer->plane(plane_idx).stride)); \
+    if (dmabufBuffer->plane(plane_idx).modifiers != DRM_FORMAT_MOD_INVALID) { \
+        attribs.append(EGL_DMA_BUF_PLANE ## plane_idx ## _MODIFIER_LO_EXT); \
+        attribs.append(EGLint(dmabufBuffer->plane(plane_idx).modifiers & 0xffffffff)); \
+        attribs.append(EGL_DMA_BUF_PLANE ## plane_idx ## _MODIFIER_HI_EXT); \
+        attribs.append(EGLint(dmabufBuffer->plane(plane_idx).modifiers >> 32)); \
+    } \
+}
+
+    switch (dmabufBuffer->planesNumber()) {
+    case 4:
+        ADD_PLANE_ATTRIBS(3);
+        Q_FALLTHROUGH();
+    case 3:
+        ADD_PLANE_ATTRIBS(2);
+        Q_FALLTHROUGH();
+    case 2:
+        ADD_PLANE_ATTRIBS(1);
+        Q_FALLTHROUGH();
+    case 1:
+        ADD_PLANE_ATTRIBS(0);
         break;
-        default:
-            return false;
-        }
+    default:
+        qCWarning(qLcWaylandCompositorHardwareIntegration) << "Buffer uses invalid number of planes:" << dmabufBuffer->planesNumber();
+        return false;
+    }
 
-        // note: EGLImageKHR does NOT take ownership of the file descriptors
-        EGLImageKHR image = egl_create_image(m_eglDisplay,
-                                             EGL_NO_CONTEXT,
-                                             EGL_LINUX_DMA_BUF_EXT,
-                                             (EGLClientBuffer) nullptr,
-                                             attribs.constData());
+    attribs.append(EGL_NONE);
 
-        if (image == EGL_NO_IMAGE_KHR) {
-            qCWarning(qLcWaylandCompositorHardwareIntegration) << "failed to create EGL image for plane" << i;
-            success = false;
-        }
+    // note: EGLImageKHR does NOT take ownership of the file descriptors
+    EGLImageKHR image = egl_create_image(m_eglDisplay,
+                                         EGL_NO_CONTEXT,
+                                         EGL_LINUX_DMA_BUF_EXT,
+                                         (EGLClientBuffer) nullptr,
+                                         attribs.constData());
 
-        dmabufBuffer->initImage(i, image);
+    if (image == EGL_NO_IMAGE_KHR) {
+        qCWarning(qLcWaylandCompositorHardwareIntegration) << "failed to create EGL image from" <<
+            dmabufBuffer->planesNumber() << "plane(s)";
+        success = false;
     }
+
+    dmabufBuffer->initImage(0, image);
+
     return success;
 }