xcb: fix regression with remote X11 clients

There were several issues here:

We were attempting to use MIT-SHM functions over SSH connection,
which is not supported. X server should detect this and return with
an appropriate error message. It does actually return BadAccess for
non-fd code path, but Qt was stubbornly trying to repeat this action
and always falling back to malloc (during window resizing). For fd
code path we were hitting X server bug, which would result in window
freeze [1].

During the initialization we check if xcb_shm_attach_checked() fails,
and disable MIT-SHM if it does. We use this logic to detect if we
are running remotely, as there are no public APIs for it. This way
we can avoid X server bug and avoid needless calling of code path
which will _always_ fail on a remote X11 connection.

[1] https://lists.x.org/archives/xorg-devel/2018-June/057011.html

Task-number: QTBUG-68449
Task-number: QTBUG-68783
Change-Id: I7ab3dcf0f323fd53001b9f7b88c2cb10809af509
Reviewed-by: Gatis Paeglis <gatis.paeglis@qt.io>

1 files changed, 23 insertions, 8 deletions

diff --git a/src/plugins/platforms/xcb/qxcbconnection.cpp b/src/plugins/platforms/xcb/qxcbconnection.cpp
index d971de766d..8ed73bc270 100644
--- a/src/plugins/platforms/xcb/qxcbconnection.cpp
+++ b/src/plugins/platforms/xcb/qxcbconnection.cpp
@@ -55,6 +55,7 @@
 #include "qxcbsystemtraytracker.h"
 #include "qxcbglintegrationfactory.h"
 #include "qxcbglintegration.h"
+#include "qxcbbackingstore.h"
 
 #include <QSocketNotifier>
 #include <QAbstractEventDispatcher>
@@ -973,7 +974,7 @@ void QXcbConnection::printXcbError(const char *message, xcb_generic_error_t *err
     uint clamped_error_code = qMin<uint>(error->error_code, (sizeof(xcb_errors) / sizeof(xcb_errors[0])) - 1);
     uint clamped_major_code = qMin<uint>(error->major_code, (sizeof(xcb_protocol_request_codes) / sizeof(xcb_protocol_request_codes[0])) - 1);
 
-    qWarning("%s: %d (%s), sequence: %d, resource id: %d, major code: %d (%s), minor code: %d",
+    qCWarning(lcQpaXcb, "%s: %d (%s), sequence: %d, resource id: %d, major code: %d (%s), minor code: %d",
              message,
              int(error->error_code), xcb_errors[clamped_error_code],
              int(error->sequence), int(error->resource_id),
@@ -2103,20 +2104,34 @@ void QXcbConnection::initializeShm()
 {
     const xcb_query_extension_reply_t *reply = xcb_get_extension_data(m_connection, &xcb_shm_id);
     if (!reply || !reply->present) {
-        qWarning("QXcbConnection: MIT-SHM extension is not present on the X server.");
+        qCDebug(lcQpaXcb, "MIT-SHM extension is not present on the X server");
         return;
     }
-
     has_shm = true;
 
     auto shm_query = Q_XCB_REPLY(xcb_shm_query_version, m_connection);
-    if (!shm_query) {
-        qWarning("QXcbConnection: Failed to request MIT-SHM version");
-        return;
+    if (shm_query) {
+        has_shm_fd = (shm_query->major_version == 1 && shm_query->minor_version >= 2) ||
+                      shm_query->major_version > 1;
+    } else {
+        qCWarning(lcQpaXcb, "QXcbConnection: Failed to request MIT-SHM version");
     }
 
-    has_shm_fd = (shm_query->major_version == 1 && shm_query->minor_version >= 2) ||
-                  shm_query->major_version > 1;
+    qCDebug(lcQpaXcb) << "Has MIT-SHM     :" << has_shm;
+    qCDebug(lcQpaXcb) << "Has MIT-SHM FD  :" << has_shm_fd;
+
+    // Temporary disable warnings (unless running in debug mode).
+    auto logging = const_cast<QLoggingCategory*>(&lcQpaXcb());
+    bool wasEnabled = logging->isEnabled(QtMsgType::QtWarningMsg);
+    if (!logging->isEnabled(QtMsgType::QtDebugMsg))
+        logging->setEnabled(QtMsgType::QtWarningMsg, false);
+    if (!QXcbBackingStore::createSystemVShmSegment(this)) {
+        qCDebug(lcQpaXcb, "failed to create System V shared memory segment (remote "
+                          "X11 connection?), disabling SHM");
+        has_shm = has_shm_fd = false;
+    }
+    if (wasEnabled)
+        logging->setEnabled(QtMsgType::QtWarningMsg, true);
 }
 
 void QXcbConnection::initializeXFixes()