xcb: Check for SHAPE extension before using it

This change makes sure that the X11 server really supports the SHAPE
extension before it is used.

Before this, libxcb would disconnect from the server as soon as it was
told to send a SHAPE request to a server which doesn't support SHAPE. Oh
and of course it would do so without any good error message.

Change-Id: I5f9481e488925e74e62f229e9f637ef262020c04
Signed-off-by: Uli Schlachter <psychon@znc.in>
Reviewed-by: Samuel Rødal <samuel.rodal@nokia.com>

3 files changed, 34 insertions, 4 deletions

diff --git a/src/plugins/platforms/xcb/qxcbconnection.cpp b/src/plugins/platforms/xcb/qxcbconnection.cpp
index 9a68d85a63..90df387f6a 100644
--- a/src/plugins/platforms/xcb/qxcbconnection.cpp
+++ b/src/plugins/platforms/xcb/qxcbconnection.cpp
@@ -109,6 +109,8 @@ QXcbConnection::QXcbConnection(QXcbNativeInterface *nativeInterface, const char
     , m_has_support_for_dri2(false)
 #endif
     , xfixes_first_event(0)
+    , has_shape_extension(false)
+    , has_input_shape(false)
 {
     m_primaryScreen = 0;
 
@@ -175,6 +177,7 @@ QXcbConnection::QXcbConnection(QXcbNativeInterface *nativeInterface, const char
 #ifdef XCB_USE_XINPUT2_MAEMO
     initializeXInput2();
 #endif
+    initializeXShape();
 
     m_wmSupport.reset(new QXcbWMSupport(this));
     m_keyboard = new QXcbKeyboard(this);
@@ -1050,6 +1053,25 @@ void QXcbConnection::initializeXRender()
 #endif
 }
 
+void QXcbConnection::initializeXShape()
+{
+    const xcb_query_extension_reply_t *xshape_reply = xcb_get_extension_data(m_connection, &xcb_shape_id);
+    if (!xshape_reply || !xshape_reply->present)
+        return;
+
+    has_shape_extension = true;
+    xcb_shape_query_version_cookie_t cookie = xcb_shape_query_version(m_connection);
+    xcb_shape_query_version_reply_t *shape_query = xcb_shape_query_version_reply(m_connection,
+                                                                                 cookie, NULL);
+    if (!shape_query) {
+        qWarning("QXcbConnection: Failed to initialize SHAPE extension");
+    } else if (shape_query->major_version > 1 || (shape_query->major_version == 1 && shape_query->minor_version >= 1)) {
+        // The input shape is the only thing added in SHAPE 1.1
+        has_input_shape = true;
+    }
+    free(shape_query);
+}
+
 #if defined(XCB_USE_EGL)
 bool QXcbConnection::hasEgl() const
 {
diff --git a/src/plugins/platforms/xcb/qxcbconnection.h b/src/plugins/platforms/xcb/qxcbconnection.h
index 7beb41bdd7..34943bfdef 100644
--- a/src/plugins/platforms/xcb/qxcbconnection.h
+++ b/src/plugins/platforms/xcb/qxcbconnection.h
@@ -354,6 +354,8 @@ public:
     inline void setTime(xcb_timestamp_t t) { if (t > m_time) m_time = t; }
 
     bool hasXFixes() const { return xfixes_first_event > 0; }
+    bool hasXShape() const { return has_shape_extension; }
+    bool hasInputShape() const { return has_input_shape; }
 
 private slots:
     void processXcbEvents();
@@ -363,6 +365,7 @@ private:
     void sendConnectionEvent(QXcbAtom::Atom atom, uint id = 0);
     void initializeXFixes();
     void initializeXRender();
+    void initializeXShape();
 #ifdef XCB_USE_DRI2
     void initializeDri2();
 #endif
@@ -429,6 +432,9 @@ private:
     QVector<PeekFunc> m_peekFuncs;
 
     uint32_t xfixes_first_event;
+
+    bool has_shape_extension;
+    bool has_input_shape;
 };
 
 #define DISPLAY_FROM_XCB(object) ((Display *)(object->connection()->xlib_display()))
diff --git a/src/plugins/platforms/xcb/qxcbdrag.cpp b/src/plugins/platforms/xcb/qxcbdrag.cpp
index a0736d81f1..3aeaaba2d8 100644
--- a/src/plugins/platforms/xcb/qxcbdrag.cpp
+++ b/src/plugins/platforms/xcb/qxcbdrag.cpp
@@ -216,7 +216,7 @@ translateCoordinates(QXcbConnection *c, xcb_window_t from, xcb_window_t to, int
 }
 
 static
-bool windowInteractsWithPosition(xcb_connection_t *connection, const QPoint & pos, xcb_window_t w, int shapeType)
+bool windowInteractsWithPosition(xcb_connection_t *connection, const QPoint & pos, xcb_window_t w, xcb_shape_sk_t shapeType)
 {
     bool interacts = true;
     xcb_shape_get_rectangles_reply_t *reply = xcb_shape_get_rectangles_reply(connection, xcb_shape_get_rectangles(connection, w, shapeType), NULL);
@@ -263,9 +263,11 @@ xcb_window_t QXcbDrag::findRealWindow(const QPoint & pos, xcb_window_t w, int md
                 free(reply);
                 if (isAware) {
                     // When ShapeInput and ShapeBounding are not set they return a single rectangle with the geometry of the window, this is why we
-                    // need an && here so that in the case one is set and the other is not we still get the correct result.
-                    windowContainsMouse = windowInteractsWithPosition(xcb_connection(), pos, w, XCB_SHAPE_SK_INPUT) &&
-                                          windowInteractsWithPosition(xcb_connection(), pos, w, XCB_SHAPE_SK_BOUNDING);
+                    // need to check both here so that in the case one is set and the other is not we still get the correct result.
+                    if (connection()->hasInputShape())
+                        windowContainsMouse = windowInteractsWithPosition(xcb_connection(), pos, w, XCB_SHAPE_SK_INPUT);
+                    if (windowContainsMouse && connection()->hasXShape())
+                        windowContainsMouse = windowInteractsWithPosition(xcb_connection(), pos, w, XCB_SHAPE_SK_BOUNDING);
                     if (windowContainsMouse)
                         return w;
                 }