Share and enable shader disk cache in QRhi OpenGL backend

The expectation for it is to function identically to what we get with
QOpenGLShaderProgram. (same environment variables, same logging
categories, etc.).

QOpenGLProgramBinaryCache is now shared between the QOpenGL convenience
classes (like QOpenGLShaderProgram) and QRhi. To achieve more modularity
and to prepare for QOpenGLShaderProgram and friends moving out of QtGui,
this class cannot depend on QOpenGLShader* anymore. This involves adding
some minor conversions between QRhi and QOpenGL enums for example.

Change-Id: I2f4664e074823ea536281aea8006a6db159a7381
Reviewed-by: Christian Strømme <christian.stromme@qt.io>

3 files changed, 148 insertions, 105 deletions

diff --git a/src/gui/opengl/qopenglprogrambinarycache.cpp b/src/gui/opengl/qopenglprogrambinarycache.cpp
index 1495471457..72bdacf43f 100644
--- a/src/gui/opengl/qopenglprogrambinarycache.cpp
+++ b/src/gui/opengl/qopenglprogrambinarycache.cpp
@@ -44,6 +44,7 @@
 #include <QStandardPaths>
 #include <QDir>
 #include <QSaveFile>
+#include <QCoreApplication>
 #include <QLoggingCategory>
 #include <QCryptographicHash>
 
@@ -54,7 +55,7 @@
 
 QT_BEGIN_NAMESPACE
 
-Q_DECLARE_LOGGING_CATEGORY(DBG_SHADER_CACHE)
+Q_LOGGING_CATEGORY(lcOpenGLProgramDiskCache, "qt.opengl.diskcache")
 
 #ifndef GL_CONTEXT_LOST
 #define GL_CONTEXT_LOST                   0x0507
@@ -64,6 +65,10 @@ Q_DECLARE_LOGGING_CATEGORY(DBG_SHADER_CACHE)
 #define GL_PROGRAM_BINARY_LENGTH          0x8741
 #endif
 
+#ifndef GL_NUM_PROGRAM_BINARY_FORMATS
+#define GL_NUM_PROGRAM_BINARY_FORMATS     0x87FE
+#endif
+
 const quint32 BINSHADER_MAGIC = 0x5174;
 const quint32 BINSHADER_VERSION = 0x3;
 const quint32 BINSHADER_QTVERSION = QT_VERSION;
@@ -123,7 +128,7 @@ QOpenGLProgramBinaryCache::QOpenGLProgramBinaryCache()
         m_cacheDir = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + subPath;
         m_cacheWritable = qt_ensureWritableDir(m_cacheDir);
     }
-    qCDebug(DBG_SHADER_CACHE, "Cache location '%s' writable = %d", qPrintable(m_cacheDir), m_cacheWritable);
+    qCDebug(lcOpenGLProgramDiskCache, "Cache location '%s' writable = %d", qPrintable(m_cacheDir), m_cacheWritable);
 }
 
 QString QOpenGLProgramBinaryCache::cacheFileName(const QByteArray &cacheKey) const
@@ -154,24 +159,24 @@ static inline QByteArray readStr(const uchar **p)
 bool QOpenGLProgramBinaryCache::verifyHeader(const QByteArray &buf) const
 {
     if (buf.size() < BASE_HEADER_SIZE) {
-        qCDebug(DBG_SHADER_CACHE, "Cached size too small");
+        qCDebug(lcOpenGLProgramDiskCache, "Cached size too small");
         return false;
     }
     const uchar *p = reinterpret_cast<const uchar *>(buf.constData());
     if (readUInt(&p) != BINSHADER_MAGIC) {
-        qCDebug(DBG_SHADER_CACHE, "Magic does not match");
+        qCDebug(lcOpenGLProgramDiskCache, "Magic does not match");
         return false;
     }
     if (readUInt(&p) != BINSHADER_VERSION) {
-        qCDebug(DBG_SHADER_CACHE, "Version does not match");
+        qCDebug(lcOpenGLProgramDiskCache, "Version does not match");
         return false;
     }
     if (readUInt(&p) != BINSHADER_QTVERSION) {
-        qCDebug(DBG_SHADER_CACHE, "Qt version does not match");
+        qCDebug(lcOpenGLProgramDiskCache, "Qt version does not match");
         return false;
     }
     if (readUInt(&p) != sizeof(quintptr)) {
-        qCDebug(DBG_SHADER_CACHE, "Architecture does not match");
+        qCDebug(lcOpenGLProgramDiskCache, "Architecture does not match");
         return false;
     }
     return true;
@@ -196,7 +201,7 @@ bool QOpenGLProgramBinaryCache::setProgramBinary(uint programId, uint blobFormat
 
     GLenum err = funcs->glGetError();
     if (err != GL_NO_ERROR) {
-        qCDebug(DBG_SHADER_CACHE, "Program binary failed to load for program %u, size %d, "
+        qCDebug(lcOpenGLProgramDiskCache, "Program binary failed to load for program %u, size %d, "
                                   "format 0x%x, err = 0x%x",
                 programId, blobSize, blobFormat, err);
         return false;
@@ -204,13 +209,13 @@ bool QOpenGLProgramBinaryCache::setProgramBinary(uint programId, uint blobFormat
     GLint linkStatus = 0;
     funcs->glGetProgramiv(programId, GL_LINK_STATUS, &linkStatus);
     if (linkStatus != GL_TRUE) {
-        qCDebug(DBG_SHADER_CACHE, "Program binary failed to load for program %u, size %d, "
+        qCDebug(lcOpenGLProgramDiskCache, "Program binary failed to load for program %u, size %d, "
                                   "format 0x%x, linkStatus = 0x%x, err = 0x%x",
                 programId, blobSize, blobFormat, linkStatus, err);
         return false;
     }
 
-    qCDebug(DBG_SHADER_CACHE, "Program binary set for program %u, size %d, format 0x%x, err = 0x%x",
+    qCDebug(lcOpenGLProgramDiskCache, "Program binary set for program %u, size %d, format 0x%x, err = 0x%x",
             programId, blobSize, blobFormat, err);
     return true;
 }
@@ -318,19 +323,19 @@ bool QOpenGLProgramBinaryCache::load(const QByteArray &cacheKey, uint programId)
     if (vendor != info.glvendor) {
         // readStr returns non-null terminated strings just pointing to inside
         // 'p' so must print these via the stream qCDebug and not constData().
-        qCDebug(DBG_SHADER_CACHE) << "GL_VENDOR does not match" << vendor << info.glvendor;
+        qCDebug(lcOpenGLProgramDiskCache) << "GL_VENDOR does not match" << vendor << info.glvendor;
         undertaker.setActive();
         return false;
     }
     QByteArray renderer = readStr(&p);
     if (renderer != info.glrenderer) {
-        qCDebug(DBG_SHADER_CACHE) << "GL_RENDERER does not match" << renderer << info.glrenderer;
+        qCDebug(lcOpenGLProgramDiskCache) << "GL_RENDERER does not match" << renderer << info.glrenderer;
         undertaker.setActive();
         return false;
     }
     QByteArray version = readStr(&p);
     if (version != info.glversion) {
-        qCDebug(DBG_SHADER_CACHE) <<  "GL_VERSION does not match" << version << info.glversion;
+        qCDebug(lcOpenGLProgramDiskCache) <<  "GL_VERSION does not match" << version << info.glversion;
         undertaker.setActive();
         return false;
     }
@@ -383,7 +388,7 @@ void QOpenGLProgramBinaryCache::save(const QByteArray &cacheKey, uint programId)
 
     const int totalSize = headerSize + paddingSize + blobSize;
 
-    qCDebug(DBG_SHADER_CACHE, "Program binary is %d bytes, err = 0x%x, total %d", blobSize, funcs->glGetError(), totalSize);
+    qCDebug(lcOpenGLProgramDiskCache, "Program binary is %d bytes, err = 0x%x, total %d", blobSize, funcs->glGetError(), totalSize);
     if (!blobSize)
         return;
 
@@ -417,7 +422,7 @@ void QOpenGLProgramBinaryCache::save(const QByteArray &cacheKey, uint programId)
 #endif
     funcs->glGetProgramBinary(programId, blobSize, &outSize, &blobFormat, p);
     if (blobSize != outSize) {
-        qCDebug(DBG_SHADER_CACHE, "glGetProgramBinary returned size %d instead of %d", outSize, blobSize);
+        qCDebug(lcOpenGLProgramDiskCache, "glGetProgramBinary returned size %d instead of %d", outSize, blobSize);
         return;
     }
 
@@ -433,9 +438,9 @@ void QOpenGLProgramBinaryCache::save(const QByteArray &cacheKey, uint programId)
     if (f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
         if (f.write(blob) < blob.length())
 #endif
-            qCDebug(DBG_SHADER_CACHE, "Failed to write %s to shader cache", qPrintable(f.fileName()));
+            qCDebug(lcOpenGLProgramDiskCache, "Failed to write %s to shader cache", qPrintable(f.fileName()));
     } else {
-        qCDebug(DBG_SHADER_CACHE, "Failed to create %s in shader cache", qPrintable(f.fileName()));
+        qCDebug(lcOpenGLProgramDiskCache, "Failed to create %s in shader cache", qPrintable(f.fileName()));
     }
 }
 
@@ -452,4 +457,45 @@ void QOpenGLProgramBinaryCache::initializeProgramBinaryOES(QOpenGLContext *conte
 }
 #endif
 
+QOpenGLProgramBinarySupportCheck::QOpenGLProgramBinarySupportCheck(QOpenGLContext *context)
+    : QOpenGLSharedResource(context->shareGroup()),
+      m_supported(false)
+{
+    if (QCoreApplication::testAttribute(Qt::AA_DisableShaderDiskCache)) {
+        qCDebug(lcOpenGLProgramDiskCache, "Shader cache disabled via app attribute");
+        return;
+    }
+    if (qEnvironmentVariableIntValue("QT_DISABLE_SHADER_DISK_CACHE")) {
+        qCDebug(lcOpenGLProgramDiskCache, "Shader cache disabled via env var");
+        return;
+    }
+
+    QOpenGLContext *ctx = QOpenGLContext::currentContext();
+    if (ctx) {
+        if (ctx->isOpenGLES()) {
+            qCDebug(lcOpenGLProgramDiskCache, "OpenGL ES v%d context", ctx->format().majorVersion());
+            if (ctx->format().majorVersion() >= 3) {
+                m_supported = true;
+            } else {
+                const bool hasExt = ctx->hasExtension("GL_OES_get_program_binary");
+                qCDebug(lcOpenGLProgramDiskCache, "GL_OES_get_program_binary support = %d", hasExt);
+                if (hasExt)
+                    m_supported = true;
+            }
+        } else {
+            const bool hasExt = ctx->hasExtension("GL_ARB_get_program_binary");
+            qCDebug(lcOpenGLProgramDiskCache, "GL_ARB_get_program_binary support = %d", hasExt);
+            if (hasExt)
+                m_supported = true;
+        }
+        if (m_supported) {
+            GLint fmtCount = 0;
+            ctx->functions()->glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &fmtCount);
+            qCDebug(lcOpenGLProgramDiskCache, "Supported binary format count = %d", fmtCount);
+            m_supported = fmtCount > 0;
+        }
+    }
+    qCDebug(lcOpenGLProgramDiskCache, "Shader cache supported = %d", m_supported);
+}
+
 QT_END_NAMESPACE
diff --git a/src/gui/opengl/qopenglprogrambinarycache_p.h b/src/gui/opengl/qopenglprogrambinarycache_p.h
index fb01e61872..f1cf24cd87 100644
--- a/src/gui/opengl/qopenglprogrambinarycache_p.h
+++ b/src/gui/opengl/qopenglprogrambinarycache_p.h
@@ -52,21 +52,26 @@
 //
 
 #include <QtGui/qtguiglobal.h>
-#include <QtGui/qopenglshaderprogram.h>
 #include <QtCore/qcache.h>
 #include <QtCore/qmutex.h>
+#include <QtGui/private/qopenglcontext_p.h>
+#include <QtGui/private/qshader_p.h>
 
 QT_BEGIN_NAMESPACE
 
+// These classes are also used by the OpenGL backend of QRhi. They must
+// therefore stay independent from QOpenGLShader(Program). Must rely only on
+// QOpenGLContext/Functions.
+
 class QOpenGLProgramBinaryCache
 {
 public:
     struct ShaderDesc {
         ShaderDesc() { }
-        ShaderDesc(QOpenGLShader::ShaderType type, const QByteArray &source = QByteArray())
-          : type(type), source(source)
+        ShaderDesc(QShader::Stage stage, const QByteArray &source = QByteArray())
+          : stage(stage), source(source)
         { }
-        QOpenGLShader::ShaderType type;
+        QShader::Stage stage;
         QByteArray source;
     };
     struct ProgramDesc {
@@ -104,6 +109,36 @@ private:
     QMutex m_mutex;
 };
 
+// While unlikely, one application can in theory use contexts with different versions
+// or profiles. Therefore any version- or extension-specific checks must be done on a
+// per-context basis, not just once per process. QOpenGLSharedResource enables this,
+// although it's once-per-sharing-context-group, not per-context. Still, this should
+// be good enough in practice.
+class QOpenGLProgramBinarySupportCheck : public QOpenGLSharedResource
+{
+public:
+    QOpenGLProgramBinarySupportCheck(QOpenGLContext *context);
+    void invalidateResource() override { }
+    void freeResource(QOpenGLContext *) override { }
+
+    bool isSupported() const { return m_supported; }
+
+private:
+    bool m_supported;
+};
+
+class QOpenGLProgramBinarySupportCheckWrapper
+{
+public:
+    QOpenGLProgramBinarySupportCheck *get(QOpenGLContext *context)
+    {
+        return m_resource.value<QOpenGLProgramBinarySupportCheck>(context);
+    }
+
+private:
+    QOpenGLMultiGroupSharedResource m_resource;
+};
+
 QT_END_NAMESPACE
 
 #endif
diff --git a/src/gui/opengl/qopenglshaderprogram.cpp b/src/gui/opengl/qopenglshaderprogram.cpp
index 153a5dd9ee..4986ca573d 100644
--- a/src/gui/opengl/qopenglshaderprogram.cpp
+++ b/src/gui/opengl/qopenglshaderprogram.cpp
@@ -47,7 +47,6 @@
 #include <QtCore/qvarlengtharray.h>
 #include <QtCore/qvector.h>
 #include <QtCore/qloggingcategory.h>
-#include <QtCore/qcoreapplication.h>
 #include <QtGui/qtransform.h>
 #include <QtGui/QColor>
 #include <QtGui/QSurfaceFormat>
@@ -178,7 +177,7 @@ QT_BEGIN_NAMESPACE
            (requires OpenGL >= 4.3 or OpenGL ES >= 3.1).
 */
 
-Q_LOGGING_CATEGORY(DBG_SHADER_CACHE, "qt.opengl.diskcache")
+Q_DECLARE_LOGGING_CATEGORY(lcOpenGLProgramDiskCache)
 
 // For GLES 3.1/3.2
 #ifndef GL_GEOMETRY_SHADER
@@ -209,10 +208,6 @@ Q_LOGGING_CATEGORY(DBG_SHADER_CACHE, "qt.opengl.diskcache")
 #define GL_PATCH_DEFAULT_INNER_LEVEL  0x8E73
 #endif
 
-#ifndef GL_NUM_PROGRAM_BINARY_FORMATS
-#define GL_NUM_PROGRAM_BINARY_FORMATS     0x87FE
-#endif
-
 #ifndef QT_OPENGL_ES_2
 static inline bool isFormatGLES(const QSurfaceFormat &f)
 {
@@ -1080,6 +1075,44 @@ bool QOpenGLShaderProgram::addCacheableShaderFromSourceCode(QOpenGLShader::Shade
     return addCacheableShaderFromSourceCode(type, QByteArray(source));
 }
 
+static inline QShader::Stage qt_shaderTypeToStage(QOpenGLShader::ShaderType type)
+{
+    switch (type) {
+    case QOpenGLShader::Vertex:
+        return QShader::VertexStage;
+    case QOpenGLShader::Fragment:
+        return QShader::FragmentStage;
+    case QOpenGLShader::Geometry:
+        return QShader::GeometryStage;
+    case QOpenGLShader::TessellationControl:
+        return QShader::TessellationControlStage;
+    case QOpenGLShader::TessellationEvaluation:
+        return QShader::TessellationEvaluationStage;
+    case QOpenGLShader::Compute:
+        return QShader::ComputeStage;
+    }
+    return QShader::VertexStage;
+}
+
+static inline QOpenGLShader::ShaderType qt_shaderStageToType(QShader::Stage stage)
+{
+    switch (stage) {
+    case QShader::VertexStage:
+        return QOpenGLShader::Vertex;
+    case QShader::TessellationControlStage:
+        return QOpenGLShader::TessellationControl;
+    case QShader::TessellationEvaluationStage:
+        return QOpenGLShader::TessellationEvaluation;
+    case QShader::GeometryStage:
+        return QOpenGLShader::Geometry;
+    case QShader::FragmentStage:
+        return QOpenGLShader::Fragment;
+    case QShader::ComputeStage:
+        return QOpenGLShader::Compute;
+    }
+    return QOpenGLShader::Vertex;
+}
+
 /*!
     \overload
 
@@ -1108,7 +1141,7 @@ bool QOpenGLShaderProgram::addCacheableShaderFromSourceCode(QOpenGLShader::Shade
     if (d->isCacheDisabled())
         return addShaderFromSourceCode(type, source);
 
-    d->binaryProgram.shaders.append(QOpenGLProgramBinaryCache::ShaderDesc(type, source));
+    d->binaryProgram.shaders.append(QOpenGLProgramBinaryCache::ShaderDesc(qt_shaderTypeToStage(type), source));
     return true;
 }
 
@@ -1165,7 +1198,7 @@ bool QOpenGLShaderProgram::addCacheableShaderFromSourceFile(QOpenGLShader::Shade
     if (d->isCacheDisabled())
         return addShaderFromSourceFile(type, fileName);
 
-    QOpenGLProgramBinaryCache::ShaderDesc shader(type);
+    QOpenGLProgramBinaryCache::ShaderDesc shader(qt_shaderTypeToStage(type));
     // NB! It could be tempting to defer reading the file contents and just
     // hash the filename as the cache key, perhaps combined with last-modified
     // timestamp checks. However, this would raise a number of issues (no
@@ -3719,77 +3752,6 @@ bool QOpenGLShader::hasOpenGLShaders(ShaderType type, QOpenGLContext *context)
     return true;
 }
 
-// While unlikely, one application can in theory use contexts with different versions
-// or profiles. Therefore any version- or extension-specific checks must be done on a
-// per-context basis, not just once per process. QOpenGLSharedResource enables this,
-// although it's once-per-sharing-context-group, not per-context. Still, this should
-// be good enough in practice.
-class QOpenGLProgramBinarySupportCheck : public QOpenGLSharedResource
-{
-public:
-    QOpenGLProgramBinarySupportCheck(QOpenGLContext *context);
-    void invalidateResource() override { }
-    void freeResource(QOpenGLContext *) override { }
-
-    bool isSupported() const { return m_supported; }
-
-private:
-    bool m_supported;
-};
-
-QOpenGLProgramBinarySupportCheck::QOpenGLProgramBinarySupportCheck(QOpenGLContext *context)
-    : QOpenGLSharedResource(context->shareGroup()),
-      m_supported(false)
-{
-    if (QCoreApplication::testAttribute(Qt::AA_DisableShaderDiskCache)) {
-        qCDebug(DBG_SHADER_CACHE, "Shader cache disabled via app attribute");
-        return;
-    }
-    if (qEnvironmentVariableIntValue("QT_DISABLE_SHADER_DISK_CACHE")) {
-        qCDebug(DBG_SHADER_CACHE, "Shader cache disabled via env var");
-        return;
-    }
-
-    QOpenGLContext *ctx = QOpenGLContext::currentContext();
-    if (ctx) {
-        if (ctx->isOpenGLES()) {
-            qCDebug(DBG_SHADER_CACHE, "OpenGL ES v%d context", ctx->format().majorVersion());
-            if (ctx->format().majorVersion() >= 3) {
-                m_supported = true;
-            } else {
-                const bool hasExt = ctx->hasExtension("GL_OES_get_program_binary");
-                qCDebug(DBG_SHADER_CACHE, "GL_OES_get_program_binary support = %d", hasExt);
-                if (hasExt)
-                    m_supported = true;
-            }
-        } else {
-            const bool hasExt = ctx->hasExtension("GL_ARB_get_program_binary");
-            qCDebug(DBG_SHADER_CACHE, "GL_ARB_get_program_binary support = %d", hasExt);
-            if (hasExt)
-                m_supported = true;
-        }
-        if (m_supported) {
-            GLint fmtCount = 0;
-            ctx->functions()->glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &fmtCount);
-            qCDebug(DBG_SHADER_CACHE, "Supported binary format count = %d", fmtCount);
-            m_supported = fmtCount > 0;
-        }
-    }
-    qCDebug(DBG_SHADER_CACHE, "Shader cache supported = %d", m_supported);
-}
-
-class QOpenGLProgramBinarySupportCheckWrapper
-{
-public:
-    QOpenGLProgramBinarySupportCheck *get(QOpenGLContext *context)
-    {
-        return m_resource.value<QOpenGLProgramBinarySupportCheck>(context);
-    }
-
-private:
-    QOpenGLMultiGroupSharedResource m_resource;
-};
-
 bool QOpenGLShaderProgramPrivate::isCacheDisabled() const
 {
     static QOpenGLProgramBinarySupportCheckWrapper binSupportCheck;
@@ -3800,7 +3762,7 @@ bool QOpenGLShaderProgramPrivate::compileCacheable()
 {
     Q_Q(QOpenGLShaderProgram);
     for (const QOpenGLProgramBinaryCache::ShaderDesc &shader : qAsConst(binaryProgram.shaders)) {
-        QScopedPointer<QOpenGLShader> s(new QOpenGLShader(shader.type, q));
+        QScopedPointer<QOpenGLShader> s(new QOpenGLShader(qt_shaderStageToType(shader.stage), q));
         if (!s->compileSourceCode(shader.source)) {
             log = s->log();
             return false;
@@ -3819,19 +3781,19 @@ bool QOpenGLShaderProgramPrivate::linkBinary()
     Q_Q(QOpenGLShaderProgram);
 
     const QByteArray cacheKey = binaryProgram.cacheKey();
-    if (DBG_SHADER_CACHE().isEnabled(QtDebugMsg))
-        qCDebug(DBG_SHADER_CACHE, "program with %d shaders, cache key %s",
+    if (lcOpenGLProgramDiskCache().isEnabled(QtDebugMsg))
+        qCDebug(lcOpenGLProgramDiskCache, "program with %d shaders, cache key %s",
                 binaryProgram.shaders.count(), cacheKey.constData());
 
     bool needsCompile = true;
     if (binCache.load(cacheKey, q->programId())) {
-        qCDebug(DBG_SHADER_CACHE, "Program binary received from cache");
+        qCDebug(lcOpenGLProgramDiskCache, "Program binary received from cache");
         needsCompile = false;
     }
 
     bool needsSave = false;
     if (needsCompile) {
-        qCDebug(DBG_SHADER_CACHE, "Program binary not in cache, compiling");
+        qCDebug(lcOpenGLProgramDiskCache, "Program binary not in cache, compiling");
         if (compileCacheable())
             needsSave = true;
         else