Use WAYLAND_DISPLAY and DISPLAY instead of XDG_SESSION_TYPE

XDG_SESSION_TYPE is a non-standard part of systemd, and not set if you
run a compositor from the command line, for instance.

[ChangeLog][Wayland] XDG_SESSION_TYPE is no longer used to determine which
platform plugin to use. Instead, if WAYLAND_DISPLAY is set in the environment,
wayland is used. Similarly, if DISPLAY is set, xcb is used. If both are
detected, wayland will be attempted first, then xcb.

Gnome-shell is still skipped for automatic wayland detection.

Fixes: QTBUG-75732
Change-Id: Ieed123330662dc29eafa31148a9b99ba0810de90
Reviewed-by: Paul Olav Tvete <paul.tvete@qt.io>

1 files changed, 35 insertions, 18 deletions

diff --git a/src/gui/kernel/qguiapplication.cpp b/src/gui/kernel/qguiapplication.cpp
index a0ab112c8e..701cd06a09 100644
--- a/src/gui/kernel/qguiapplication.cpp
+++ b/src/gui/kernel/qguiapplication.cpp
@@ -1394,26 +1394,43 @@ void QGuiApplicationPrivate::createPlatformIntegration()
     platformName = QT_QPA_DEFAULT_PLATFORM_NAME;
 #endif
 #if defined(Q_OS_UNIX) && !defined(Q_OS_DARWIN)
-    QByteArray sessionType = qgetenv("XDG_SESSION_TYPE");
-    if (!sessionType.isEmpty()) {
-        if (sessionType == QByteArrayLiteral("x11") && !platformName.contains(QByteArrayLiteral("xcb"))) {
-            platformName = QByteArrayLiteral("xcb");
-        } else if (sessionType == QByteArrayLiteral("wayland") && !platformName.contains(QByteArrayLiteral("wayland"))) {
-            QByteArray currentDesktop = qgetenv("XDG_CURRENT_DESKTOP").toLower();
-            QByteArray sessionDesktop = qgetenv("XDG_SESSION_DESKTOP").toLower();
-            if (currentDesktop.contains("gnome") || sessionDesktop.contains("gnome")) {
-                qInfo() << "Warning: Ignoring XDG_SESSION_TYPE=wayland on Gnome."
-                        << "Use QT_QPA_PLATFORM=wayland to run on Wayland anyway.";
-            } else {
-                platformName = QByteArrayLiteral("wayland");
-            }
+    QList<QByteArray> platformArguments = platformName.split(':');
+    QByteArray platformPluginBase = platformArguments.first();
+
+    const bool hasWaylandDisplay = qEnvironmentVariableIsSet("WAYLAND_DISPLAY");
+    const bool isWaylandSessionType = qgetenv("XDG_SESSION_TYPE") == "wayland";
+
+    QVector<QByteArray> preferredPlatformOrder;
+    const bool defaultIsXcb = platformPluginBase == "xcb";
+    const QByteArray xcbPlatformName = defaultIsXcb ? platformName : "xcb";
+    if (qEnvironmentVariableIsSet("DISPLAY")) {
+        preferredPlatformOrder << xcbPlatformName;
+        if (defaultIsXcb)
+            platformName.clear();
+    }
+
+    const bool defaultIsWayland = !defaultIsXcb && platformPluginBase.startsWith("wayland");
+    const QByteArray waylandPlatformName = defaultIsWayland ? platformName : "wayland";
+    if (hasWaylandDisplay || isWaylandSessionType) {
+        const QByteArray currentDesktop = qgetenv("XDG_CURRENT_DESKTOP").toLower();
+        const QByteArray sessionDesktop = qgetenv("XDG_SESSION_DESKTOP").toLower();
+        const bool isGnome = currentDesktop.contains("gnome") || sessionDesktop.contains("gnome");
+        if (isGnome) {
+            qInfo() << "Warning: Ignoring WAYLAND_DISPLAY on Gnome."
+                    << "Use QT_QPA_PLATFORM=wayland to run on Wayland anyway.";
+            preferredPlatformOrder.append(waylandPlatformName);
+        } else {
+            preferredPlatformOrder.prepend(waylandPlatformName);
         }
+
+        if (defaultIsWayland)
+            platformName.clear();
     }
-#ifdef QT_QPA_DEFAULT_PLATFORM_NAME
-    // Add it as fallback in case XDG_SESSION_TYPE is something wrong
-    if (!platformName.contains(QT_QPA_DEFAULT_PLATFORM_NAME))
-        platformName += QByteArrayLiteral(";" QT_QPA_DEFAULT_PLATFORM_NAME);
-#endif
+
+    if (!platformName.isEmpty())
+        preferredPlatformOrder.append(platformName);
+
+    platformName = preferredPlatformOrder.join(';');
 #endif
 
     QByteArray platformNameEnv = qgetenv("QT_QPA_PLATFORM");