Add caching mechanism to ShaderBuilder

It is a 2 level cache system: runtime and offline

First we now generate a hash key based on unique features of a graph,
the api it targets, the last time the graph file was modified.

We then use that hash key to at runtime check whether we have already
loaded a graph matching the same hash key.

If that's not the case, we switch to using the offline cache and checking
whether a file already exists for that shader or not. If the file exists,
we load it, use the code it contains and add it to the runtime cache.

If the file does not exist, we revert to actually generating the shader
and we save the generated code to both the offline cache and the runtime cache.

By default, the offline cache with try to write into the location reported by
QStandardLocation::writableLocation(QStandardLocation::TempLocation).

Optionally, the environment variable QT3D_WRITABLE_CACHE_PATH can be set with
a path to a writable location.

If QT3D_REBUILD_SHADER_CACHE is not empty, cache will be regenerated.
Alternatively if QT3D_DISABLE_SHADER_CACHE is set, cache will be ignored.

Pick-to: 5.15
Change-Id: Ia348f92ce4cdd5e63ec89e58b7954d1f127f26bb
Reviewed-by: Mike Krus <mike.krus@kdab.com>

3 files changed, 112 insertions, 5 deletions

diff --git a/src/render/backend/abstractrenderer_p.h b/src/render/backend/abstractrenderer_p.h
index c0fd9dd73..b7d9b43b1 100644
--- a/src/render/backend/abstractrenderer_p.h
+++ b/src/render/backend/abstractrenderer_p.h
@@ -196,6 +196,15 @@ public:
     virtual QSurfaceFormat format() = 0;
     virtual QOpenGLContext *shareContext() const = 0;
     virtual const GraphicsApiFilterData *contextInfo() const = 0;
+
+    // Runtime Cache for Generated Shader Graph
+    bool containsGeneratedShaderGraph(const QByteArray &key) const { return m_cachedGeneratedShaderGraphes.contains(key); };
+    QByteArray cachedGeneratedShaderGraph(const QByteArray &key) const { return m_cachedGeneratedShaderGraphes.value(key); };
+    void insertGeneratedShaderGraph(const QByteArray &key, const QByteArray shaderCode) { m_cachedGeneratedShaderGraphes.insert(key, shaderCode); }
+    void removeGeneratedShaderGraph(const QByteArray &key) { m_cachedGeneratedShaderGraphes.remove(key); };
+
+private:
+    QHash<QByteArray, QByteArray> m_cachedGeneratedShaderGraphes;
 };
 
 Q_DECLARE_OPERATORS_FOR_FLAGS(AbstractRenderer::BackendNodeDirtySet)
diff --git a/src/render/materialsystem/shaderbuilder.cpp b/src/render/materialsystem/shaderbuilder.cpp
index c7a2aa347..84efcdaaa 100644
--- a/src/render/materialsystem/shaderbuilder.cpp
+++ b/src/render/materialsystem/shaderbuilder.cpp
@@ -52,6 +52,10 @@
 #include <QFile>
 #include <QFileInfo>
 #include <QUrl>
+#include <QCryptographicHash>
+#include <QDateTime>
+#include <QStandardPaths>
+#include <QDir>
 
 static void initResources()
 {
@@ -220,6 +224,53 @@ void ShaderBuilder::generateCode(QShaderProgram::ShaderType type)
         return;
     }
 
+    auto updateShaderCodeAndClearDirty = [&] (const QByteArray &shaderCode) {
+        m_codes.insert(type, shaderCode);
+        m_dirtyTypes.remove(type);
+        m_pendingUpdates.push_back({ peerId(),
+                                     type,
+                                     m_codes.value(type) });
+    };
+
+    const QByteArray cacheKey = hashKeyForShaderGraph(type);
+    const bool forceRegenerate = qEnvironmentVariableIsSet("QT3D_REBUILD_SHADER_CACHE");
+    const bool useCache = !qEnvironmentVariableIsSet("QT3D_DISABLE_SHADER_CACHE") || !forceRegenerate;
+    const QByteArray userProvidedPath = qgetenv("QT3D_WRITABLE_CACHE_PATH");
+    const QString cachedFilterPath = QDir(userProvidedPath.isEmpty() ?
+                                              QStandardPaths::writableLocation(QStandardPaths::TempLocation)
+                                            : QString::fromUtf8(userProvidedPath)).absoluteFilePath(QString::fromUtf8(cacheKey) + QLatin1String(".qt3d"));
+    QFile cachedShaderFile(cachedFilterPath);
+
+    // Check our runtime cache to see if we have already loaded the shader previously
+    if (useCache) {
+        // We check if we already have generated a shader previously for the
+        // given type, the given graph, the given API and the current set of layer
+        // If that's the case it's faster to load the pre generated shader file
+
+        if (m_renderer && m_renderer->containsGeneratedShaderGraph(cacheKey)) {
+            updateShaderCodeAndClearDirty(m_renderer->cachedGeneratedShaderGraph(cacheKey));
+            return;
+        }
+
+        // else check if a cachedShader file exists
+        if (cachedShaderFile.exists()) {
+            if (!cachedShaderFile.open(QFile::ReadOnly)) {
+                qWarning() << "Couldn't open cached shader file:" << graphPath;
+                // Too bad, we have to generate the shader below
+            } else {
+                // Use cached shader
+                const QByteArray shaderCode = cachedShaderFile.readAll();
+                updateShaderCodeAndClearDirty(shaderCode);
+
+                // Record to runtime cache
+                if (m_renderer)
+                    m_renderer->insertGeneratedShaderGraph(cacheKey, shaderCode);
+                return;
+            }
+        }
+    }
+
+    // Generate Shader and Cache the result for subsequent uses
     auto graphLoader = QShaderGraphLoader();
     graphLoader.setPrototypes(qt3dGlobalShaderPrototypes->prototypes());
     graphLoader.setDevice(&file);
@@ -247,12 +298,20 @@ void ShaderBuilder::generateCode(QShaderProgram::ShaderType type)
 
     const auto code = generator.createShaderCode(m_enabledLayers);
     const auto deincludified = QShaderProgramPrivate::deincludify(code, graphPath + QStringLiteral(".glsl"));
-    m_codes.insert(type, deincludified);
-    m_dirtyTypes.remove(type);
 
-    m_pendingUpdates.push_back({ peerId(),
-                                 type,
-                                 m_codes.value(type) });
+    updateShaderCodeAndClearDirty(deincludified);
+
+    // Record to runtime cache
+    if (useCache || forceRegenerate) {
+        if (m_renderer)
+            m_renderer->insertGeneratedShaderGraph(cacheKey, deincludified);
+
+        // Record to file cache
+        if (cachedShaderFile.open(QFile::WriteOnly))
+            cachedShaderFile.write(deincludified);
+        else
+            qWarning() << "Unable to write cached shader file";
+    }
 }
 
 void ShaderBuilder::syncFromFrontEnd(const QNode *frontEnd, bool firstTime)
@@ -297,6 +356,43 @@ void ShaderBuilder::syncFromFrontEnd(const QNode *frontEnd, bool firstTime)
     }
 }
 
+QByteArray ShaderBuilder::hashKeyForShaderGraph(QShaderProgram::ShaderType type) const
+{
+    const auto graphPath = Qt3DCore::QUrlHelper::urlToLocalFileOrQrc(shaderGraph(type));
+    QFile file(graphPath);
+    if (!file.exists()) {
+        qWarning() << graphPath << "doesn't exist";
+        return {};
+    }
+
+    QCryptographicHash hashBuilder(QCryptographicHash::Sha1);
+    // Add graphPath
+    hashBuilder.addData(graphPath.toUtf8());
+    // Get TimeStamp and Graph file size
+    QFileInfo info(graphPath);
+    const QString fileInfo = QString::fromUtf8("%1_%2")
+            .arg(info.lastModified().toSecsSinceEpoch())
+            .arg(info.size());
+    hashBuilder.addData(fileInfo.toUtf8());
+
+    // Add Layers
+    for (const QString &layer : m_enabledLayers)
+        hashBuilder.addData(layer.toUtf8());
+
+    // Add GraphicsInfo
+    const QString graphicsInfo = QString::fromUtf8("API: %1 Profile: %2 Major: %3 Minor: %4")
+            .arg(int(m_graphicsApi.m_api))
+            .arg(int(m_graphicsApi.m_profile))
+            .arg(int(m_graphicsApi.m_major))
+            .arg(int(m_graphicsApi.m_minor));
+    hashBuilder.addData(graphicsInfo.toUtf8());
+
+    // Add Shader Type
+    hashBuilder.addData(QString::number(type).toUtf8());
+
+    return hashBuilder.result().toHex();
+}
+
 } // namespace Render
 } // namespace Qt3DRender
 
diff --git a/src/render/materialsystem/shaderbuilder_p.h b/src/render/materialsystem/shaderbuilder_p.h
index 3297c5d7b..b15aa89db 100644
--- a/src/render/materialsystem/shaderbuilder_p.h
+++ b/src/render/materialsystem/shaderbuilder_p.h
@@ -97,6 +97,8 @@ public:
 
     std::vector<ShaderBuilderUpdate> &&takePendingUpdates() { return std::move(m_pendingUpdates); }
 
+    QByteArray hashKeyForShaderGraph(QShaderProgram::ShaderType type) const;
+
 private:
     void setEnabledLayers(const QStringList &layers);