Lighthouse xcb and xlib: Add support for transparency of GLX windows.

Merge-request: 1231
Reviewed-by: Samuel Rødal <samuel.rodal@nokia.com>
(cherry picked from commit a3b627e1c5ce03a2500ab35c64729b1995639dcc)

10 files changed, 78 insertions, 25 deletions

diff --git a/src/plugins/platforms/glxconvenience/glxconvenience.pri b/src/plugins/platforms/glxconvenience/glxconvenience.pri
index d6c992229e..b4d43a30b5 100644
--- a/src/plugins/platforms/glxconvenience/glxconvenience.pri
+++ b/src/plugins/platforms/glxconvenience/glxconvenience.pri
@@ -5,3 +5,11 @@ HEADERS += \
 
 SOURCES += \
     $$PWD/qglxconvenience.cpp
+
+CONFIG += xrender
+
+xrender {
+	LIBS += -lXrender
+} else {
+	DEFINES += QT_NO_XRENDER
+}
diff --git a/src/plugins/platforms/glxconvenience/qglxconvenience.cpp b/src/plugins/platforms/glxconvenience/qglxconvenience.cpp
index 7cee3e2ea1..eb0f6a6604 100644
--- a/src/plugins/platforms/glxconvenience/qglxconvenience.cpp
+++ b/src/plugins/platforms/glxconvenience/qglxconvenience.cpp
@@ -43,6 +43,10 @@
 
 #include <QtCore/QVector>
 
+#ifndef QT_NO_XRENDER
+#include <X11/extensions/Xrender.h>
+#endif
+
 enum {
     XFocusOut = FocusOut,
     XFocusIn = FocusIn,
@@ -84,14 +88,15 @@ QVector<int> qglx_buildSpec(const QPlatformWindowFormat &format, int drawableBit
                 spec[i++] = GLX_ALPHA_SIZE; spec[i++] = (format.alphaBufferSize() == -1) ? 1 : format.alphaBufferSize();
             }
 
-        spec[i++] = GLX_ACCUM_RED_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
-        spec[i++] = GLX_ACCUM_GREEN_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
-        spec[i++] = GLX_ACCUM_BLUE_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
+        if (format.accum()) {
+            spec[i++] = GLX_ACCUM_RED_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
+            spec[i++] = GLX_ACCUM_GREEN_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
+            spec[i++] = GLX_ACCUM_BLUE_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
 
-        if (format.alpha()) {
-            spec[i++] = GLX_ACCUM_ALPHA_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
+            if (format.alpha()) {
+                spec[i++] = GLX_ACCUM_ALPHA_SIZE; spec[i++] = (format.accumBufferSize() == -1) ? 1 : format.accumBufferSize();
+            }
         }
-
     } else {
         spec[i++] = GLX_RENDER_TYPE; spec[i++] = GLX_COLOR_INDEX_BIT; //I'm really not sure if this works....
         spec[i++] = GLX_BUFFER_SIZE; spec[i++] = 8;
@@ -136,8 +141,17 @@ GLXFBConfig qglx_findConfig(Display *display, int screen , const QPlatformWindow
                 if (reducedFormat.alpha()) {
                     int alphaSize;
                     glXGetFBConfigAttrib(display,configs[i],GLX_ALPHA_SIZE,&alphaSize);
-                    if (alphaSize > 0)
-                        break;
+                    if (alphaSize > 0) {
+                        XVisualInfo *visual = glXGetVisualFromFBConfig(display, chosenConfig);
+#if !defined(QT_NO_XRENDER)
+                        XRenderPictFormat *pictFormat = XRenderFindVisualFormat(display, visual->visual);
+                        if (pictFormat->direct.alphaMask > 0)
+                            break;
+#else
+                        if (visual->depth == 32)
+                            break;
+#endif
+                    }
                 } else {
                     break; // Just choose the first in the list if there's no alpha requested
                 }
diff --git a/src/plugins/platforms/wayland/gl_integration/xcomposite_glx/qwaylandxcompositeglxcontext.cpp b/src/plugins/platforms/wayland/gl_integration/xcomposite_glx/qwaylandxcompositeglxcontext.cpp
index caf51170a6..baf0fa3432 100644
--- a/src/plugins/platforms/wayland/gl_integration/xcomposite_glx/qwaylandxcompositeglxcontext.cpp
+++ b/src/plugins/platforms/wayland/gl_integration/xcomposite_glx/qwaylandxcompositeglxcontext.cpp
@@ -132,10 +132,12 @@ void QWaylandXCompositeGLXContext::geometryChanged()
     Colormap cmap = XCreateColormap(mGlxIntegration->xDisplay(),mGlxIntegration->rootWindow(),visualInfo->visual,AllocNone);
 
     XSetWindowAttributes a;
+    a.background_pixel = WhitePixel(mGlxIntegration->xDisplay(), mGlxIntegration->screen());
+    a.border_pixel = BlackPixel(mGlxIntegration->xDisplay(), mGlxIntegration->screen());
     a.colormap = cmap;
     mXWindow = XCreateWindow(mGlxIntegration->xDisplay(), mGlxIntegration->rootWindow(),0, 0, size.width(), size.height(),
                              0, visualInfo->depth, InputOutput, visualInfo->visual,
-                             CWColormap, &a);
+                             CWBackPixel|CWBorderPixel|CWColormap, &a);
 
     XCompositeRedirectWindow(mGlxIntegration->xDisplay(), mXWindow, CompositeRedirectManual);
     XMapWindow(mGlxIntegration->xDisplay(), mXWindow);
diff --git a/src/plugins/platforms/xcb/qxcbwindow.cpp b/src/plugins/platforms/xcb/qxcbwindow.cpp
index 0456638ab9..3365c22c39 100644
--- a/src/plugins/platforms/xcb/qxcbwindow.cpp
+++ b/src/plugins/platforms/xcb/qxcbwindow.cpp
@@ -131,13 +131,17 @@ QXcbWindow::QXcbWindow(QWidget *tlw)
         visualInfo = XGetVisualInfo(DISPLAY_FROM_XCB(this), VisualIDMask, &visualInfoTemplate, &matchingCount);
 #endif //XCB_USE_GLX
         if (visualInfo) {
+            m_depth = visualInfo->depth;
+            m_format = (m_depth == 32) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
             Colormap cmap = XCreateColormap(DISPLAY_FROM_XCB(this), m_screen->root(), visualInfo->visual, AllocNone);
 
             XSetWindowAttributes a;
+            a.background_pixel = WhitePixel(DISPLAY_FROM_XCB(this), m_screen->screenNumber());
+            a.border_pixel = BlackPixel(DISPLAY_FROM_XCB(this), m_screen->screenNumber());
             a.colormap = cmap;
             m_window = XCreateWindow(DISPLAY_FROM_XCB(this), m_screen->root(), tlw->x(), tlw->y(), tlw->width(), tlw->height(),
                                       0, visualInfo->depth, InputOutput, visualInfo->visual,
-                                      CWColormap, &a);
+                                      CWBackPixel|CWBorderPixel|CWColormap, &a);
 
             printf("created GL window: %d\n", m_window);
         } else {
@@ -147,6 +151,8 @@ QXcbWindow::QXcbWindow(QWidget *tlw)
 #endif //defined(XCB_USE_GLX) || defined(XCB_USE_EGL)
     {
         m_window = xcb_generate_id(xcb_connection());
+        m_depth = m_screen->screen()->root_depth;
+        m_format = (m_depth == 32) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
 
         Q_XCB_CALL(xcb_create_window(xcb_connection(),
                                      XCB_COPY_FROM_PARENT,            // depth -- same as root
diff --git a/src/plugins/platforms/xcb/qxcbwindow.h b/src/plugins/platforms/xcb/qxcbwindow.h
index e049837253..20b464061a 100644
--- a/src/plugins/platforms/xcb/qxcbwindow.h
+++ b/src/plugins/platforms/xcb/qxcbwindow.h
@@ -44,6 +44,7 @@
 
 #include <QtGui/QPlatformWindow>
 #include <QtGui/QPlatformWindowFormat>
+#include <QtGui/QImage>
 
 #include <xcb/xcb.h>
 #include <xcb/sync.h>
@@ -74,6 +75,8 @@ public:
     QPlatformGLContext *glContext() const;
 
     xcb_window_t window() const { return m_window; }
+    uint depth() const { return m_depth; }
+    QImage::Format format() const { return m_format; }
 
     void handleExposeEvent(const xcb_expose_event_t *event);
     void handleClientMessageEvent(const xcb_client_message_event_t *event);
@@ -99,6 +102,9 @@ private:
     xcb_window_t m_window;
     QPlatformGLContext *m_context;
 
+    uint m_depth;
+    QImage::Format m_format;
+
     xcb_sync_int64_t m_syncValue;
     xcb_sync_counter_t m_syncCounter;
 
diff --git a/src/plugins/platforms/xcb/qxcbwindowsurface.cpp b/src/plugins/platforms/xcb/qxcbwindowsurface.cpp
index 718f09383c..38c01f9cfd 100644
--- a/src/plugins/platforms/xcb/qxcbwindowsurface.cpp
+++ b/src/plugins/platforms/xcb/qxcbwindowsurface.cpp
@@ -58,7 +58,7 @@
 class QXcbShmImage : public QXcbObject
 {
 public:
-    QXcbShmImage(QXcbScreen *connection, const QSize &size);
+    QXcbShmImage(QXcbScreen *connection, const QSize &size, uint depth, QImage::Format format);
     ~QXcbShmImage() { destroy(); }
 
     QImage *image() { return &m_qimage; }
@@ -81,7 +81,7 @@ private:
     QRegion m_dirty;
 };
 
-QXcbShmImage::QXcbShmImage(QXcbScreen *screen, const QSize &size)
+QXcbShmImage::QXcbShmImage(QXcbScreen *screen, const QSize &size, uint depth, QImage::Format format)
     : QXcbObject(screen->connection())
     , m_gc(0)
     , m_gc_window(0)
@@ -91,7 +91,7 @@ QXcbShmImage::QXcbShmImage(QXcbScreen *screen, const QSize &size)
                                           size.width(),
                                           size.height(),
                                           XCB_IMAGE_FORMAT_Z_PIXMAP,
-                                          screen->depth(),
+                                          depth,
                                           0,
                                           ~0,
                                           0);
@@ -111,7 +111,7 @@ QXcbShmImage::QXcbShmImage(QXcbScreen *screen, const QSize &size)
     if (shmctl(m_shm_info.shmid, IPC_RMID, 0) == -1)
         qWarning() << "QXcbWindowSurface: Error while marking the shared memory segment to be destroyed";
 
-    m_qimage = QImage( (uchar*) m_xcb_image->data, m_xcb_image->width, m_xcb_image->height, m_xcb_image->stride, screen->format());
+    m_qimage = QImage( (uchar*) m_xcb_image->data, m_xcb_image->width, m_xcb_image->height, m_xcb_image->stride, format);
 }
 
 void QXcbShmImage::destroy()
@@ -232,9 +232,10 @@ void QXcbWindowSurface::resize(const QSize &size)
     QWindowSurface::resize(size);
 
     QXcbScreen *screen = static_cast<QXcbScreen *>(QPlatformScreen::platformScreenForWidget(window()));
+    QXcbWindow* win = static_cast<QXcbWindow *>(window()->platformWindow());
 
     delete m_image;
-    m_image = new QXcbShmImage(screen, size);
+    m_image = new QXcbShmImage(screen, size, win->depth(), win->format());
     Q_XCB_NOOP(connection());
 
     m_syncingResize = true;
diff --git a/src/plugins/platforms/xlib/qxlibintegration.cpp b/src/plugins/platforms/xlib/qxlibintegration.cpp
index 78f907abd2..b0c8bc9560 100644
--- a/src/plugins/platforms/xlib/qxlibintegration.cpp
+++ b/src/plugins/platforms/xlib/qxlibintegration.cpp
@@ -150,7 +150,7 @@ bool QXlibIntegration::hasOpenGL() const
 {
 #if !defined(QT_NO_OPENGL)
 #if !defined(QT_OPENGL_ES_2)
-    QXlibScreen *screen = static_cast<const QXlibScreen *>(mScreens.at(0));
+    QXlibScreen *screen = static_cast<QXlibScreen *>(mScreens.at(0));
     return glXQueryExtension(screen->display()->nativeDisplay(), 0, 0) != 0;
 #else
     static bool eglHasbeenInitialized = false;
diff --git a/src/plugins/platforms/xlib/qxlibwindow.cpp b/src/plugins/platforms/xlib/qxlibwindow.cpp
index 90bcc0d9d5..4efcb1101c 100644
--- a/src/plugins/platforms/xlib/qxlibwindow.cpp
+++ b/src/plugins/platforms/xlib/qxlibwindow.cpp
@@ -102,18 +102,27 @@ QXlibWindow::QXlibWindow(QWidget *window)
         visualInfo = XGetVisualInfo(mScreen->display()->nativeDisplay(), VisualIDMask, &visualInfoTemplate, &matchingCount);
 #endif //!defined(QT_OPENGL_ES_2)
         if (visualInfo) {
-            Colormap cmap = XCreateColormap(mScreen->display()->nativeDisplay(),mScreen->rootWindow(),visualInfo->visual,AllocNone);
+            mDepth = visualInfo->depth;
+            mFormat = (mDepth == 32) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
+            mVisual = visualInfo->visual;
+            Colormap cmap = XCreateColormap(mScreen->display()->nativeDisplay(), mScreen->rootWindow(), visualInfo->visual, AllocNone);
 
             XSetWindowAttributes a;
+            a.background_pixel = WhitePixel(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber());
+            a.border_pixel = BlackPixel(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber());
             a.colormap = cmap;
             x_window = XCreateWindow(mScreen->display()->nativeDisplay(), mScreen->rootWindow(),x, y, w, h,
                                      0, visualInfo->depth, InputOutput, visualInfo->visual,
-                                     CWColormap, &a);
+                                     CWBackPixel|CWBorderPixel|CWColormap, &a);
         } else {
             qFatal("no window!");
         }
 #endif //!defined(QT_NO_OPENGL)
     } else {
+        mDepth = mScreen->depth();
+        mFormat = (mDepth == 32) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
+        mVisual = mScreen->defaultVisual();
+
         x_window = XCreateSimpleWindow(mScreen->display()->nativeDisplay(), mScreen->rootWindow(),
                                        x, y, w, h, 0 /*border_width*/,
                                        mScreen->blackPixel(), mScreen->whitePixel());
diff --git a/src/plugins/platforms/xlib/qxlibwindow.h b/src/plugins/platforms/xlib/qxlibwindow.h
index 08694a5026..2d11224c23 100644
--- a/src/plugins/platforms/xlib/qxlibwindow.h
+++ b/src/plugins/platforms/xlib/qxlibwindow.h
@@ -122,6 +122,10 @@ public:
     Window xWindow() const;
     GC graphicsContext() const;
 
+    inline uint depth() const { return mDepth; }
+    QImage::Format format() const { return mFormat; }
+    Visual* visual() const { return mVisual; }
+
 protected:
     QVector<Atom> getNetWmState() const;
     void setMWMHints(const QXlibMWMHints &mwmhints);
@@ -135,6 +139,10 @@ private:
     Window x_window;
     GC gc;
 
+    uint mDepth;
+    QImage::Format mFormat;
+    Visual* mVisual;
+
     GC createGC();
 
     QPlatformGLContext *mGLContext;
diff --git a/src/plugins/platforms/xlib/qxlibwindowsurface.cpp b/src/plugins/platforms/xlib/qxlibwindowsurface.cpp
index 513f10dc6a..78fe387bb8 100644
--- a/src/plugins/platforms/xlib/qxlibwindowsurface.cpp
+++ b/src/plugins/platforms/xlib/qxlibwindowsurface.cpp
@@ -80,20 +80,19 @@ void QXlibShmImageInfo::destroy()
 
 void QXlibWindowSurface::resizeShmImage(int width, int height)
 {
+    QXlibScreen *screen = QXlibScreen::testLiteScreenForWidget(window());
+    QXlibWindow *win = static_cast<QXlibWindow*>(window()->platformWindow());
 
 #ifdef DONT_USE_MIT_SHM
-    shm_img = QImage(width, height, QImage::Format_RGB32);
+    shm_img = QImage(width, height, win->format());
 #else
 
-    QXlibScreen *screen = QXlibScreen::testLiteScreenForWidget(window());
     if (image_info)
         image_info->destroy();
     else
         image_info = new QXlibShmImageInfo(screen->display()->nativeDisplay());
 
-    Visual *visual = screen->defaultVisual();
-
-    XImage *image = XShmCreateImage (screen->display()->nativeDisplay(), visual, 24, ZPixmap, 0,
+    XImage *image = XShmCreateImage (screen->display()->nativeDisplay(), win->visual(), win->depth(), ZPixmap, 0,
                                      &image_info->shminfo, width, height);
 
 
@@ -160,11 +159,11 @@ void QXlibWindowSurface::flush(QWidget *widget, const QRegion &region, const QPo
 #ifdef DONT_USE_MIT_SHM
     // just convert the image every time...
     if (!shm_img.isNull()) {
-        Visual *visual = DefaultVisual(screen->display(), screen->xScreenNumber());
+        QXlibWindow *win = static_cast<QXlibWindow*>(window()->platformWindow());
 
         QImage image = shm_img;
         //img.convertToFormat(
-        XImage *xi = XCreateImage(screen->display(), visual, 24, ZPixmap,
+        XImage *xi = XCreateImage(screen->display(), win->visual(), win->depth(), ZPixmap,
                                   0, (char *) image.scanLine(0), image.width(), image.height(),
                                   32, image.bytesPerLine());