Fall back on glXChooseVisual() if glXChooseFBConfig() doesn't work.

Some older drivers don't fully support glXChooseFBConfig(). As a bonus,
fix some memory leaks here and there.

Task-number: QTBUG-21880
Change-Id: Ie306dee27f616927a6aa55fd71601569b828afcc
Reviewed-by: Uli Schlachter <psychon@znc.in>
Reviewed-by: Jørgen Lind <jorgen.lind@nokia.com>

5 files changed, 135 insertions, 55 deletions

diff --git a/src/platformsupport/glxconvenience/qglxconvenience.cpp b/src/platformsupport/glxconvenience/qglxconvenience.cpp
index ea433f5a33..0d48c4e16a 100644
--- a/src/platformsupport/glxconvenience/qglxconvenience.cpp
+++ b/src/platformsupport/glxconvenience/qglxconvenience.cpp
@@ -42,6 +42,7 @@
 #include "qglxconvenience_p.h"
 
 #include <QtCore/QVector>
+#include <QtCore/QVarLengthArray>
 
 #ifndef QT_NO_XRENDER
 #include <X11/extensions/Xrender.h>
@@ -131,14 +132,19 @@ GLXFBConfig qglx_findConfig(Display *display, int screen , const QSurfaceFormat
                     glXGetFBConfigAttrib(display,configs[i],GLX_ALPHA_SIZE,&alphaSize);
                     if (alphaSize > 0) {
                         XVisualInfo *visual = glXGetVisualFromFBConfig(display, chosenConfig);
+                        bool hasAlpha = false;
+
 #if !defined(QT_NO_XRENDER)
                         XRenderPictFormat *pictFormat = XRenderFindVisualFormat(display, visual->visual);
-                        if (pictFormat->direct.alphaMask > 0)
-                            break;
+                        hasAlpha = pictFormat->direct.alphaMask > 0;
 #else
-                        if (visual->depth == 32)
-                            break;
+                        hasAlpha = visual->depth == 32;
 #endif
+
+                        XFree(visual);
+
+                        if (hasAlpha)
+                            break;
                     }
                 } else {
                     break; // Just choose the first in the list if there's no alpha requested
@@ -150,16 +156,59 @@ GLXFBConfig qglx_findConfig(Display *display, int screen , const QSurfaceFormat
         reducedFormat = qglx_reduceSurfaceFormat(reducedFormat,&reduced);
     }
 
-    if (!chosenConfig)
-        qWarning("Warning: no suitable glx confiuration found");
-
     return chosenConfig;
 }
 
 XVisualInfo *qglx_findVisualInfo(Display *display, int screen, const QSurfaceFormat &format)
 {
+    XVisualInfo *visualInfo = 0;
+
     GLXFBConfig config = qglx_findConfig(display,screen,format);
-    XVisualInfo *visualInfo = glXGetVisualFromFBConfig(display,config);
+    if (config)
+        visualInfo = glXGetVisualFromFBConfig(display, config);
+
+    // attempt to fall back to glXChooseVisual
+    bool reduced = true;
+    QSurfaceFormat reducedFormat = format;
+    while (!visualInfo && reduced) {
+        QVarLengthArray<int, 13> attribs;
+        attribs.append(GLX_RGBA);
+
+        if (reducedFormat.redBufferSize() > 0) {
+            attribs.append(GLX_RED_SIZE);
+            attribs.append(reducedFormat.redBufferSize());
+        }
+
+        if (reducedFormat.greenBufferSize() > 0) {
+            attribs.append(GLX_GREEN_SIZE);
+            attribs.append(reducedFormat.greenBufferSize());
+        }
+
+        if (reducedFormat.blueBufferSize() > 0) {
+            attribs.append(GLX_BLUE_SIZE);
+            attribs.append(reducedFormat.blueBufferSize());
+        }
+
+        if (reducedFormat.stencilBufferSize() > 0) {
+            attribs.append(GLX_STENCIL_SIZE);
+            attribs.append(reducedFormat.stencilBufferSize());
+        }
+
+        if (reducedFormat.depthBufferSize() > 0) {
+            attribs.append(GLX_DEPTH_SIZE);
+            attribs.append(reducedFormat.depthBufferSize());
+        }
+
+        if (reducedFormat.swapBehavior() != QSurfaceFormat::SingleBuffer)
+            attribs.append(GLX_DOUBLEBUFFER);
+
+        attribs.append(XNone);
+
+        visualInfo = glXChooseVisual(display, screen, attribs.data());
+
+        reducedFormat = qglx_reduceSurfaceFormat(reducedFormat, &reduced);
+    }
+
     return visualInfo;
 }
 
@@ -208,7 +257,13 @@ QSurfaceFormat qglx_reduceSurfaceFormat(const QSurfaceFormat &format, bool *redu
     QSurfaceFormat retFormat = format;
     *reduced = true;
 
-    if (retFormat.samples() > 1) {
+    if (retFormat.redBufferSize() > 1) {
+        retFormat.setRedBufferSize(1);
+    } else if (retFormat.greenBufferSize() > 1) {
+        retFormat.setGreenBufferSize(1);
+    } else if (retFormat.blueBufferSize() > 1) {
+        retFormat.setBlueBufferSize(1);
+    } else if (retFormat.samples() > 1) {
         retFormat.setSamples(0);
     } else if (retFormat.stereo()) {
         retFormat.setStereo(false);
@@ -218,6 +273,8 @@ QSurfaceFormat qglx_reduceSurfaceFormat(const QSurfaceFormat &format, bool *redu
         retFormat.setAlphaBufferSize(0);
     }else if (retFormat.depthBufferSize() > 0) {
         retFormat.setDepthBufferSize(0);
+    }else if (retFormat.swapBehavior() != QSurfaceFormat::SingleBuffer) {
+        retFormat.setSwapBehavior(QSurfaceFormat::SingleBuffer);
     }else{
         *reduced = false;
     }
diff --git a/src/plugins/platforms/xcb/qglxintegration.cpp b/src/plugins/platforms/xcb/qglxintegration.cpp
index b01b9e3ea1..c8a857eaf2 100644
--- a/src/plugins/platforms/xcb/qglxintegration.cpp
+++ b/src/plugins/platforms/xcb/qglxintegration.cpp
@@ -71,15 +71,28 @@ QGLXContext::QGLXContext(QXcbScreen *screen, const QSurfaceFormat &format, QPlat
 
     GLXFBConfig config = qglx_findConfig(DISPLAY_FROM_XCB(screen),screen->screenNumber(),format);
 
-    m_context = glXCreateNewContext(DISPLAY_FROM_XCB(screen), config, GLX_RGBA_TYPE, m_shareContext, TRUE);
-    if (!m_context && m_shareContext) {
-        // re-try without a shared glx context
-        m_shareContext = 0;
-        m_context = glXCreateNewContext(DISPLAY_FROM_XCB(screen), config, GLX_RGBA_TYPE, 0, TRUE);
-    }
+    if (config) {
+        m_context = glXCreateNewContext(DISPLAY_FROM_XCB(screen), config, GLX_RGBA_TYPE, m_shareContext, TRUE);
+        if (!m_context && m_shareContext) {
+            // re-try without a shared glx context
+            m_shareContext = 0;
+            m_context = glXCreateNewContext(DISPLAY_FROM_XCB(screen), config, GLX_RGBA_TYPE, 0, TRUE);
+        }
 
-    if (m_context)
-        m_format = qglx_surfaceFormatFromGLXFBConfig(DISPLAY_FROM_XCB(screen), config, m_context);
+        if (m_context)
+            m_format = qglx_surfaceFormatFromGLXFBConfig(DISPLAY_FROM_XCB(screen), config, m_context);
+    } else {
+        XVisualInfo *visualInfo = qglx_findVisualInfo(DISPLAY_FROM_XCB(m_screen), screen->screenNumber(), format);
+        if (!visualInfo)
+            qFatal("Could not initialize GLX");
+        m_context = glXCreateContext(DISPLAY_FROM_XCB(screen), visualInfo, m_shareContext, true);
+        if (!m_context && m_shareContext) {
+            // re-try without a shared glx context
+            m_shareContext = 0;
+            m_context = glXCreateContext(DISPLAY_FROM_XCB(screen), visualInfo, 0, true);
+        }
+        XFree(visualInfo);
+    }
 }
 
 QGLXContext::~QGLXContext()
diff --git a/src/plugins/platforms/xcb/qxcbwindow.cpp b/src/plugins/platforms/xcb/qxcbwindow.cpp
index 6d1eae4675..6febe4703d 100644
--- a/src/plugins/platforms/xcb/qxcbwindow.cpp
+++ b/src/plugins/platforms/xcb/qxcbwindow.cpp
@@ -130,10 +130,8 @@ static inline QImage::Format imageFormatForDepth(int depth)
         case 32: return QImage::Format_ARGB32_Premultiplied;
         case 24: return QImage::Format_RGB32;
         case 16: return QImage::Format_RGB16;
-        default: break;
+        default: return QImage::Format_Invalid;
     }
-    qFatal("Unsupported display depth %d", depth);
-    return QImage::Format_Invalid;
 }
 
 QXcbWindow::QXcbWindow(QWindow *window)
@@ -211,7 +209,8 @@ void QXcbWindow::create()
     {
 #if defined(XCB_USE_GLX)
         XVisualInfo *visualInfo = qglx_findVisualInfo(DISPLAY_FROM_XCB(m_screen),m_screen->screenNumber(), window()->format());
-
+        if (!visualInfo)
+            qFatal("Could not initialize GLX");
 #elif defined(XCB_USE_EGL)
         EGLDisplay eglDisplay = connection()->egl_display();
         EGLConfig eglConfig = q_configFromGLFormat(eglDisplay, window()->format(), true);
@@ -224,25 +223,25 @@ void QXcbWindow::create()
         XVisualInfo *visualInfo;
         int matchingCount = 0;
         visualInfo = XGetVisualInfo(DISPLAY_FROM_XCB(this), VisualIDMask, &visualInfoTemplate, &matchingCount);
+        if (!visualInfo)
+            qFatal("Could not initialize EGL");
 #endif //XCB_USE_GLX
-        if (visualInfo) {
-            m_depth = visualInfo->depth;
-            m_imageFormat = imageFormatForDepth(m_depth);
-            Colormap cmap = XCreateColormap(DISPLAY_FROM_XCB(this), xcb_parent_id, visualInfo->visual, AllocNone);
+        m_depth = visualInfo->depth;
+        m_imageFormat = imageFormatForDepth(m_depth);
+        Colormap cmap = XCreateColormap(DISPLAY_FROM_XCB(this), xcb_parent_id, visualInfo->visual, AllocNone);
 
-            XSetWindowAttributes a;
-            a.background_pixel = WhitePixel(DISPLAY_FROM_XCB(this), m_screen->screenNumber());
-            a.border_pixel = BlackPixel(DISPLAY_FROM_XCB(this), m_screen->screenNumber());
-            a.colormap = cmap;
+        XSetWindowAttributes a;
+        a.background_pixel = WhitePixel(DISPLAY_FROM_XCB(this), m_screen->screenNumber());
+        a.border_pixel = BlackPixel(DISPLAY_FROM_XCB(this), m_screen->screenNumber());
+        a.colormap = cmap;
 
-            m_visualId = visualInfo->visualid;
+        m_visualId = visualInfo->visualid;
 
-            m_window = XCreateWindow(DISPLAY_FROM_XCB(this), xcb_parent_id, rect.x(), rect.y(), rect.width(), rect.height(),
-                                      0, visualInfo->depth, InputOutput, visualInfo->visual,
-                                      CWBackPixel|CWBorderPixel|CWColormap, &a);
-        } else {
-            qFatal("no window!");
-        }
+        m_window = XCreateWindow(DISPLAY_FROM_XCB(this), xcb_parent_id, rect.x(), rect.y(), rect.width(), rect.height(),
+                                  0, visualInfo->depth, InputOutput, visualInfo->visual,
+                                  CWBackPixel|CWBorderPixel|CWColormap, &a);
+
+        XFree(visualInfo);
     } else
 #endif //defined(XCB_USE_GLX) || defined(XCB_USE_EGL)
     {
diff --git a/src/plugins/platforms/xlib/qglxintegration.cpp b/src/plugins/platforms/xlib/qglxintegration.cpp
index 875e166672..80ea371eab 100644
--- a/src/plugins/platforms/xlib/qglxintegration.cpp
+++ b/src/plugins/platforms/xlib/qglxintegration.cpp
@@ -70,9 +70,19 @@ QGLXContext::QGLXContext(QXlibScreen *screen, const QSurfaceFormat &format, QPla
     if (share)
         shareGlxContext = static_cast<const QGLXContext*>(share)->glxContext();
 
-    GLXFBConfig config = qglx_findConfig(screen->display()->nativeDisplay(),screen->xScreenNumber(),format);
-    m_context = glXCreateNewContext(screen->display()->nativeDisplay(),config,GLX_RGBA_TYPE,shareGlxContext,TRUE);
-    m_windowFormat = qglx_surfaceFormatFromGLXFBConfig(screen->display()->nativeDisplay(),config,m_context);
+    Display *xDisplay = screen->display()->nativeDisplay();
+
+    GLXFBConfig config = qglx_findConfig(xDisplay,screen->xScreenNumber(),format);
+    if (config) {
+        m_context = glXCreateNewContext(xDisplay,config,GLX_RGBA_TYPE,shareGlxContext,TRUE);
+        m_windowFormat = qglx_surfaceFormatFromGLXFBConfig(xDisplay,config,m_context);
+    } else {
+        XVisualInfo *visualInfo = qglx_findVisualInfo(xDisplay, screen->xScreenNumber(), format);
+        if (!visualInfo)
+            qFatal("Could not initialize GLX");
+        m_context = glXCreateContext(xDisplay, visualInfo, shareGlxContext, true);
+        XFree(visualInfo);
+    }
 
 #ifdef MYX11_DEBUG
     qDebug() << "QGLXGLContext::create context" << m_context;
diff --git a/src/plugins/platforms/xlib/qxlibwindow.cpp b/src/plugins/platforms/xlib/qxlibwindow.cpp
index d5724d6050..6c9081c9ca 100644
--- a/src/plugins/platforms/xlib/qxlibwindow.cpp
+++ b/src/plugins/platforms/xlib/qxlibwindow.cpp
@@ -87,6 +87,8 @@ QXlibWindow::QXlibWindow(QWindow *window)
 #if !defined(QT_OPENGL_ES_2)
         XVisualInfo *visualInfo = qglx_findVisualInfo(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber(),
                                                       window->format());
+        if (!visualInfo)
+            qFatal("Could not initialize GLX");
 #else
         QPlatformWindowFormat windowFormat = correctColorBuffers(window->platformWindowFormat());
 
@@ -101,23 +103,22 @@ QXlibWindow::QXlibWindow(QWindow *window)
         XVisualInfo *visualInfo;
         int matchingCount = 0;
         visualInfo = XGetVisualInfo(mScreen->display()->nativeDisplay(), VisualIDMask, &visualInfoTemplate, &matchingCount);
+        if (!visualInfo)
+            qFatal("Could not initialize EGL");
 #endif //!defined(QT_OPENGL_ES_2)
-        if (visualInfo) {
-            mDepth = visualInfo->depth;
-            mFormat = (mDepth == 32) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
-            mVisual = visualInfo->visual;
-            Colormap cmap = XCreateColormap(mScreen->display()->nativeDisplay(), mScreen->rootWindow(), visualInfo->visual, AllocNone);
-
-            XSetWindowAttributes a;
-            a.background_pixel = WhitePixel(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber());
-            a.border_pixel = BlackPixel(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber());
-            a.colormap = cmap;
-            x_window = XCreateWindow(mScreen->display()->nativeDisplay(), mScreen->rootWindow(),x, y, w, h,
-                                     0, visualInfo->depth, InputOutput, visualInfo->visual,
-                                     CWBackPixel|CWBorderPixel|CWColormap, &a);
-        } else {
-            qFatal("no window!");
-        }
+        mDepth = visualInfo->depth;
+        mFormat = (mDepth == 32) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
+        mVisual = visualInfo->visual;
+        Colormap cmap = XCreateColormap(mScreen->display()->nativeDisplay(), mScreen->rootWindow(), visualInfo->visual, AllocNone);
+
+        XSetWindowAttributes a;
+        a.background_pixel = WhitePixel(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber());
+        a.border_pixel = BlackPixel(mScreen->display()->nativeDisplay(), mScreen->xScreenNumber());
+        a.colormap = cmap;
+        x_window = XCreateWindow(mScreen->display()->nativeDisplay(), mScreen->rootWindow(),x, y, w, h,
+                                 0, visualInfo->depth, InputOutput, visualInfo->visual,
+                                 CWBackPixel|CWBorderPixel|CWColormap, &a);
+        XFree(visualInfo);
     } else
 #endif //!defined(QT_NO_OPENGL)
     {