Handle endian mismatch between X11 client and server

If the server and client has different endian we need to swizzle the
image pixels, we can do that by swizzling the masks and try to match
the new configuration. This is a rather rare setup so we don't try
to match every combination.

This patch fixes the colors when running Qt in a bigendian QEMU chroot.

Change-Id: Ie83f9607563cba137b2e1a63e996a05d43ff603e
Reviewed-by: Gatis Paeglis <gatis.paeglis@qt.io>

4 files changed, 57 insertions, 25 deletions

diff --git a/src/plugins/platforms/xcb/qxcbconnection.h b/src/plugins/platforms/xcb/qxcbconnection.h
index 999dc0630c..efc5e666fc 100644
--- a/src/plugins/platforms/xcb/qxcbconnection.h
+++ b/src/plugins/platforms/xcb/qxcbconnection.h
@@ -409,6 +409,15 @@ public:
     const xcb_setup_t *setup() const { return m_setup; }
     const xcb_format_t *formatForDepth(uint8_t depth) const;
 
+    bool imageNeedsEndianSwap() const
+    {
+#if Q_BYTE_ORDER == Q_BIG_ENDIAN
+        return m_setup->image_byte_order != XCB_IMAGE_ORDER_MSB_FIRST;
+#else
+        return m_setup->image_byte_order != XCB_IMAGE_ORDER_LSB_FIRST;
+#endif
+    }
+
     QXcbKeyboard *keyboard() const { return m_keyboard; }
 
 #ifndef QT_NO_CLIPBOARD
diff --git a/src/plugins/platforms/xcb/qxcbimage.cpp b/src/plugins/platforms/xcb/qxcbimage.cpp
index 8635a03dcb..9fb0d44b74 100644
--- a/src/plugins/platforms/xcb/qxcbimage.cpp
+++ b/src/plugins/platforms/xcb/qxcbimage.cpp
@@ -38,6 +38,7 @@
 ****************************************************************************/
 
 #include "qxcbimage.h"
+#include <QtCore/QtEndian>
 #include <QtGui/QColor>
 #include <QtGui/private/qimage_p.h>
 #include <QtGui/private/qdrawhelper_p.h>
@@ -54,6 +55,7 @@ extern "C" {
 
 QT_BEGIN_NAMESPACE
 
+// TODO: Merge with imageFormatForVisual in qxcbwindow.cpp
 QImage::Format qt_xcb_imageFormatForVisual(QXcbConnection *connection, uint8_t depth,
                                            const xcb_visualtype_t *visual)
 {
@@ -82,6 +84,7 @@ QImage::Format qt_xcb_imageFormatForVisual(QXcbConnection *connection, uint8_t d
         && visual->green_mask == 0x7e0 && visual->blue_mask == 0x1f)
         return QImage::Format_RGB16;
 
+    qWarning("qt_xcb_imageFormatForVisual did not recognize format");
     return QImage::Format_Invalid;
 }
 
@@ -106,36 +109,26 @@ QPixmap qt_xcb_pixmapFromXPixmap(QXcbConnection *connection, xcb_pixmap_t pixmap
     if (format != QImage::Format_Invalid) {
         uint32_t bytes_per_line = length / height;
         QImage image(const_cast<uint8_t *>(data), width, height, bytes_per_line, format);
-        uint8_t image_byte_order = connection->setup()->image_byte_order;
 
         // we may have to swap the byte order
-        if ((QSysInfo::ByteOrder == QSysInfo::LittleEndian && image_byte_order == XCB_IMAGE_ORDER_MSB_FIRST)
-            || (QSysInfo::ByteOrder == QSysInfo::BigEndian && image_byte_order == XCB_IMAGE_ORDER_LSB_FIRST))
-        {
-            for (int i=0; i < image.height(); i++) {
-                switch (format) {
-                case QImage::Format_RGB16: {
-                    ushort *p = (ushort*)image.scanLine(i);
+        if (connection->imageNeedsEndianSwap()) {
+            if (image.depth() == 16) {
+                for (int i = 0; i < image.height(); ++i) {
+                    ushort *p = reinterpret_cast<ushort *>(image.scanLine(i));
                     ushort *end = p + image.width();
                     while (p < end) {
-                        *p = ((*p << 8) & 0xff00) | ((*p >> 8) & 0x00ff);
+                        *p = qbswap(*p);
                         p++;
                     }
-                    break;
                 }
-                case QImage::Format_RGB32: // fall-through
-                case QImage::Format_ARGB32_Premultiplied: {
-                    uint *p = (uint*)image.scanLine(i);
+            } else if (image.depth() == 32) {
+                for (int i = 0; i < image.height(); ++i) {
+                    uint *p = reinterpret_cast<uint *>(image.scanLine(i));
                     uint *end = p + image.width();
                     while (p < end) {
-                        *p = ((*p << 24) & 0xff000000) | ((*p << 8) & 0x00ff0000)
-                            | ((*p >> 8) & 0x0000ff00) | ((*p >> 24) & 0x000000ff);
+                        *p = qbswap(*p);
                         p++;
                     }
-                    break;
-                }
-                default:
-                    Q_ASSERT(false);
                 }
             }
         }
diff --git a/src/plugins/platforms/xcb/qxcbwindow.cpp b/src/plugins/platforms/xcb/qxcbwindow.cpp
index a97b951935..aaca7def99 100644
--- a/src/plugins/platforms/xcb/qxcbwindow.cpp
+++ b/src/plugins/platforms/xcb/qxcbwindow.cpp
@@ -42,6 +42,7 @@
 #include <QtDebug>
 #include <QMetaEnum>
 #include <QScreen>
+#include <QtCore/qendian.h>
 #include <QtGui/QIcon>
 #include <QtGui/QRegion>
 #include <QtGui/private/qhighdpiscaling_p.h>
@@ -176,11 +177,17 @@ static inline bool isTransient(const QWindow *w)
            || w->type() == Qt::Popup;
 }
 
-static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, quint32 blue_mask, bool *rgbSwap)
+// TODO: Merge with qt_xcb_imageFormatForVisual in qxcbimage.cpp
+QImage::Format QXcbWindow::imageFormatForVisual(const xcb_visualtype_t *visual, bool *rgbSwap) const
 {
+    const bool connectionEndianSwap = connection()->imageNeedsEndianSwap();
+    // We swap the masks and see if we can recognize it as a host format
+    const quint32 red_mask = connectionEndianSwap ? qbswap(visual->red_mask) : visual->red_mask;
+    const quint32 blue_mask = connectionEndianSwap ? qbswap(visual->blue_mask) : visual->blue_mask;
+
     if (rgbSwap)
         *rgbSwap = false;
-    switch (depth) {
+    switch (m_depth) {
     case 32:
         if (blue_mask == 0xff)
             return QImage::Format_ARGB32_Premultiplied;
@@ -189,10 +196,21 @@ static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, q
         if (blue_mask == 0x3ff)
             return QImage::Format_A2RGB30_Premultiplied;
         if (red_mask == 0xff) {
+#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
+            return QImage::Format_RGBA8888_Premultiplied;
+#else
             if (rgbSwap)
                 *rgbSwap = true;
             return QImage::Format_ARGB32_Premultiplied;
+#endif
+        }
+#if Q_BYTE_ORDER == Q_BIG_ENDIAN
+        if (red_mask == 0xff00 && blue_mask == 0xff000000) {
+            if (rgbSwap)
+                *rgbSwap = true;
+            return QImage::Format_RGBA8888_Premultiplied;
         }
+#endif
         break;
     case 30:
         if (red_mask == 0x3ff)
@@ -204,10 +222,20 @@ static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, q
         if (blue_mask == 0xff)
             return QImage::Format_RGB32;
         if (red_mask == 0xff) {
+#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
+            return QImage::Format_RGBX8888;
+#else
             if (rgbSwap)
                 *rgbSwap = true;
             return QImage::Format_RGB32;
+#endif
         }
+#if Q_BYTE_ORDER == Q_BIG_ENDIAN
+        if (red_mask == 0xff00 && blue_mask == 0xff000000) {
+            *rgbSwap = true;
+            return QImage::Format_RGBX8888;
+        }
+#endif
         break;
     case 16:
         if (blue_mask == 0x1f)
@@ -236,9 +264,10 @@ static inline QImage::Format imageFormatForVisual(int depth, quint32 red_mask, q
     default:
         break;
     }
-    qWarning("Unsupported screen format: depth: %d, red_mask: %x, blue_mask: %x", depth, red_mask, blue_mask);
+    qWarning("Unsupported screen format: depth: %d, red_mask: %x, blue_mask: %x", m_depth, red_mask, blue_mask);
 
-    switch (depth) {
+    switch (m_depth) {
+    case 32:
     case 24:
         qWarning("Using RGB32 fallback, if this works your X11 server is reporting a bad screen format.");
         return QImage::Format_RGB32;
@@ -381,7 +410,7 @@ void QXcbWindow::create()
         }
         if (!visual)
             visual = platformScreen->visualForId(m_visualId);
-        m_imageFormat = imageFormatForVisual(m_depth, visual->red_mask, visual->blue_mask, &m_imageRgbSwap);
+        m_imageFormat = imageFormatForVisual(visual, &m_imageRgbSwap);
         connection()->addWindowEventListener(m_window, this);
         return;
     }
@@ -451,7 +480,7 @@ void QXcbWindow::create()
 
     m_visualId = visual->visual_id;
     m_depth = platformScreen->depthOfVisual(m_visualId);
-    m_imageFormat = imageFormatForVisual(m_depth, visual->red_mask, visual->blue_mask, &m_imageRgbSwap);
+    m_imageFormat = imageFormatForVisual(visual, &m_imageRgbSwap);
 
     quint32 mask = XCB_CW_BACK_PIXMAP
                  | XCB_CW_BORDER_PIXEL
diff --git a/src/plugins/platforms/xcb/qxcbwindow.h b/src/plugins/platforms/xcb/qxcbwindow.h
index 1ce9b0a42f..ebace55328 100644
--- a/src/plugins/platforms/xcb/qxcbwindow.h
+++ b/src/plugins/platforms/xcb/qxcbwindow.h
@@ -188,6 +188,7 @@ public Q_SLOTS:
 protected:
     virtual void resolveFormat(const QSurfaceFormat &format) { m_format = format; }
     virtual const xcb_visualtype_t *createVisual();
+    QImage::Format imageFormatForVisual(const xcb_visualtype_t *visual, bool *rgbSwap) const;
 
     QXcbScreen *parentScreen();