Add caching mechanism to ShaderBuilder

It is a 2 level cache system: runtime and offline

First we now generate a hash key based on unique features of a graph,
the api it targets, the last time the graph file was modified.

We then use that hash key to at runtime check whether we have already
loaded a graph matching the same hash key.

If that's not the case, we switch to using the offline cache and checking
whether a file already exists for that shader or not. If the file exists,
we load it, use the code it contains and add it to the runtime cache.

If the file does not exist, we revert to actually generating the shader
and we save the generated code to both the offline cache and the runtime cache.

By default, the offline cache with try to write into the location reported by
QStandardLocation::writableLocation(QStandardLocation::TempLocation).

Optionally, the environment variable QT3D_WRITABLE_CACHE_PATH can be set with
a path to a writable location.

If QT3D_REBUILD_SHADER_CACHE is not empty, cache will be regenerated.
Alternatively if QT3D_DISABLE_SHADER_CACHE is set, cache will be ignored.

Change-Id: Ia348f92ce4cdd5e63ec89e58b7954d1f127f26bb
Reviewed-by: Mike Krus <mike.krus@kdab.com>
(cherry picked from commit 17324467897e25bfc1abef0db540b0690c04a55e)

4 files changed, 311 insertions, 7 deletions

diff --git a/src/render/backend/abstractrenderer_p.h b/src/render/backend/abstractrenderer_p.h
index 0dd24dcd8..ec08ab641 100644
--- a/src/render/backend/abstractrenderer_p.h
+++ b/src/render/backend/abstractrenderer_p.h
@@ -190,6 +190,15 @@ public:
     // These commands are executed in a dedicated command thread
     // More will be added later
     virtual void loadShader(Shader *shader, Qt3DRender::Render::HShader shaderHandle) = 0;
+
+    // Runtime Cache for Generated Shader Graph
+    bool containsGeneratedShaderGraph(const QByteArray &key) const { return m_cachedGeneratedShaderGraphes.contains(key); };
+    QByteArray cachedGeneratedShaderGraph(const QByteArray &key) const { return m_cachedGeneratedShaderGraphes.value(key); };
+    void insertGeneratedShaderGraph(const QByteArray &key, const QByteArray shaderCode) { m_cachedGeneratedShaderGraphes.insert(key, shaderCode); }
+    void removeGeneratedShaderGraph(const QByteArray &key) { m_cachedGeneratedShaderGraphes.remove(key); };
+
+private:
+    QHash<QByteArray, QByteArray> m_cachedGeneratedShaderGraphes;
 };
 
 Q_DECLARE_OPERATORS_FOR_FLAGS(AbstractRenderer::BackendNodeDirtySet)
diff --git a/src/render/materialsystem/shaderbuilder.cpp b/src/render/materialsystem/shaderbuilder.cpp
index b168756c9..d7ccea7da 100644
--- a/src/render/materialsystem/shaderbuilder.cpp
+++ b/src/render/materialsystem/shaderbuilder.cpp
@@ -54,6 +54,10 @@
 #include <QFile>
 #include <QFileInfo>
 #include <QUrl>
+#include <QCryptographicHash>
+#include <QDateTime>
+#include <QStandardPaths>
+#include <QDir>
 
 static void initResources()
 {
@@ -222,6 +226,53 @@ void ShaderBuilder::generateCode(QShaderProgram::ShaderType type)
         return;
     }
 
+    auto updateShaderCodeAndClearDirty = [&] (const QByteArray &shaderCode) {
+        m_codes.insert(type, shaderCode);
+        m_dirtyTypes.remove(type);
+        m_pendingUpdates.push_back({ peerId(),
+                                     type,
+                                     m_codes.value(type) });
+    };
+
+    const QByteArray cacheKey = hashKeyForShaderGraph(type);
+    const bool forceRegenerate = qEnvironmentVariableIsSet("QT3D_REBUILD_SHADER_CACHE");
+    const bool useCache = !qEnvironmentVariableIsSet("QT3D_DISABLE_SHADER_CACHE") || !forceRegenerate;
+    const QByteArray userProvidedPath = qgetenv("QT3D_WRITABLE_CACHE_PATH");
+    const QString cachedFilterPath = QDir(userProvidedPath.isEmpty() ?
+                                              QStandardPaths::writableLocation(QStandardPaths::TempLocation)
+                                            : QString::fromUtf8(userProvidedPath)).absoluteFilePath(QString::fromUtf8(cacheKey) + QLatin1String(".qt3d"));
+    QFile cachedShaderFile(cachedFilterPath);
+
+    // Check our runtime cache to see if we have already loaded the shader previously
+    if (useCache) {
+        // We check if we already have generated a shader previously for the
+        // given type, the given graph, the given API and the current set of layer
+        // If that's the case it's faster to load the pre generated shader file
+
+        if (m_renderer && m_renderer->containsGeneratedShaderGraph(cacheKey)) {
+            updateShaderCodeAndClearDirty(m_renderer->cachedGeneratedShaderGraph(cacheKey));
+            return;
+        }
+
+        // else check if a cachedShader file exists
+        if (cachedShaderFile.exists()) {
+            if (!cachedShaderFile.open(QFile::ReadOnly)) {
+                qWarning() << "Couldn't open cached shader file:" << graphPath;
+                // Too bad, we have to generate the shader below
+            } else {
+                // Use cached shader
+                const QByteArray shaderCode = cachedShaderFile.readAll();
+                updateShaderCodeAndClearDirty(shaderCode);
+
+                // Record to runtime cache
+                if (m_renderer)
+                    m_renderer->insertGeneratedShaderGraph(cacheKey, shaderCode);
+                return;
+            }
+        }
+    }
+
+    // Generate Shader and Cache the result for subsequent uses
     auto graphLoader = QShaderGraphLoader();
     graphLoader.setPrototypes(qt3dGlobalShaderPrototypes->prototypes());
     graphLoader.setDevice(&file);
@@ -249,12 +300,20 @@ void ShaderBuilder::generateCode(QShaderProgram::ShaderType type)
 
     const auto code = generator.createShaderCode(m_enabledLayers);
     const auto deincludified = QShaderProgramPrivate::deincludify(code, graphPath + QStringLiteral(".glsl"));
-    m_codes.insert(type, deincludified);
-    m_dirtyTypes.remove(type);
 
-    m_pendingUpdates.push_back({ peerId(),
-                                 type,
-                                 m_codes.value(type) });
+    updateShaderCodeAndClearDirty(deincludified);
+
+    // Record to runtime cache
+    if (useCache || forceRegenerate) {
+        if (m_renderer)
+            m_renderer->insertGeneratedShaderGraph(cacheKey, deincludified);
+
+        // Record to file cache
+        if (cachedShaderFile.open(QFile::WriteOnly))
+            cachedShaderFile.write(deincludified);
+        else
+            qWarning() << "Unable to write cached shader file";
+    }
 }
 
 void ShaderBuilder::syncFromFrontEnd(const QNode *frontEnd, bool firstTime)
@@ -299,6 +358,43 @@ void ShaderBuilder::syncFromFrontEnd(const QNode *frontEnd, bool firstTime)
     }
 }
 
+QByteArray ShaderBuilder::hashKeyForShaderGraph(QShaderProgram::ShaderType type) const
+{
+    const auto graphPath = Qt3DRender::QUrlHelper::urlToLocalFileOrQrc(shaderGraph(type));
+    QFile file(graphPath);
+    if (!file.exists()) {
+        qWarning() << graphPath << "doesn't exist";
+        return {};
+    }
+
+    QCryptographicHash hashBuilder(QCryptographicHash::Sha1);
+    // Add graphPath
+    hashBuilder.addData(graphPath.toUtf8());
+    // Get TimeStamp and Graph file size
+    QFileInfo info(graphPath);
+    const QString fileInfo = QString::fromUtf8("%1_%2")
+            .arg(info.lastModified().toSecsSinceEpoch())
+            .arg(info.size());
+    hashBuilder.addData(fileInfo.toUtf8());
+
+    // Add Layers
+    for (const QString &layer : m_enabledLayers)
+        hashBuilder.addData(layer.toUtf8());
+
+    // Add GraphicsInfo
+    const QString graphicsInfo = QString::fromUtf8("API: %1 Profile: %2 Major: %3 Minor: %4")
+            .arg(int(m_graphicsApi.m_api))
+            .arg(int(m_graphicsApi.m_profile))
+            .arg(int(m_graphicsApi.m_major))
+            .arg(int(m_graphicsApi.m_minor));
+    hashBuilder.addData(graphicsInfo.toUtf8());
+
+    // Add Shader Type
+    hashBuilder.addData(QString::number(type).toUtf8());
+
+    return hashBuilder.result().toHex();
+}
+
 } // namespace Render
 } // namespace Qt3DRender
 
diff --git a/src/render/materialsystem/shaderbuilder_p.h b/src/render/materialsystem/shaderbuilder_p.h
index 0a799afaa..2aaad882b 100644
--- a/src/render/materialsystem/shaderbuilder_p.h
+++ b/src/render/materialsystem/shaderbuilder_p.h
@@ -97,6 +97,8 @@ public:
 
     QVector<ShaderBuilderUpdate> takePendingUpdates() { return std::move(m_pendingUpdates); }
 
+    QByteArray hashKeyForShaderGraph(QShaderProgram::ShaderType type) const;
+
 private:
     void setEnabledLayers(const QStringList &layers);
 
diff --git a/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp b/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp
index 9be7db5f1..2bab9d6f5 100644
--- a/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp
+++ b/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp
@@ -272,6 +272,7 @@ private slots:
     void shouldHandleEnabledLayersPropertyChange()
     {
         // GIVEN
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
         Qt3DRender::Render::ShaderBuilder backend;
         Qt3DRender::QShaderProgramBuilder frontend;
         TestRenderer renderer;
@@ -469,6 +470,7 @@ private slots:
     void shouldHandleShaderCodeGeneration()
     {
         // GIVEN
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
         Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
         QVERIFY(!Qt3DRender::Render::ShaderBuilder::getPrototypeNames().isEmpty());
 
@@ -556,8 +558,7 @@ private slots:
     void checkCodeUpdatedNotification()
     {
         // GIVEN
-        QSKIP("Disabled for Qt Base QShaderGenerator Integration");
-
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
         Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
         QVERIFY(!Qt3DRender::Render::ShaderBuilder::getPrototypeNames().isEmpty());
         QFETCH(Qt3DRender::QShaderProgram::ShaderType, type);
@@ -606,6 +607,202 @@ private slots:
         QVERIFY(!backend.isShaderCodeDirty(type));
         QCOMPARE(backend.shaderCode(type), gl3Code);
     }
+
+    void checkFileCaching()
+    {
+        // GIVEN
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        const auto gl2Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::NoProfile;
+            api.m_major = 2;
+            api.m_minor = 0;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+        qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8());
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+        QByteArray hashKey;
+        {
+            // WHEN
+            Qt3DRender::Render::ShaderBuilder b;
+            b.setGraphicsApi(gl3Api);
+            const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+            b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+            b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+            // THEN
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+
+            hashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex);
+            QCOMPARE(hashKey.length(), 40);
+
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(),
+                     QString::fromUtf8(hashKey) + QLatin1String(".qt3d"));
+        }
+        {
+            // WHEN
+            Qt3DRender::Render::ShaderBuilder b;
+            b.setGraphicsApi(gl3Api);
+            const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+            b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+            b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+            // THEN
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(),
+                     QString::fromUtf8(hashKey) + QLatin1String(".qt3d"));
+        }
+        {
+            // WHEN
+            Qt3DRender::Render::ShaderBuilder b;
+            b.setGraphicsApi(gl2Api);
+            const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+            b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+            b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+            QByteArray gl2HashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex);
+
+            // THEN
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 2);
+            QVERIFY(gl2HashKey != hashKey);
+        }
+    }
+
+    void checkRuntimeCaching()
+    {
+        // GIVEN
+        TestRenderer renderer;
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+        qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8());
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+
+        // WHEN
+        Qt3DRender::Render::ShaderBuilder b;
+        b.setGraphicsApi(gl3Api);
+        const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+        b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+
+        const QByteArray hashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex);
+        QCOMPARE(hashKey.length(), 40);
+
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(),
+                 QString::fromUtf8(hashKey) + QLatin1String(".qt3d"));
+
+        QVERIFY(!renderer.containsGeneratedShaderGraph(hashKey));
+
+        // WHEN
+        b.setRenderer(&renderer);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN
+        QVERIFY(renderer.containsGeneratedShaderGraph(hashKey));
+    }
+
+    void checkDontUseCache()
+    {
+        // GIVEN
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+
+        // WHEN
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
+        Qt3DRender::Render::ShaderBuilder b;
+        b.setGraphicsApi(gl3Api);
+        const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+        b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 0);
+    }
+
+    void checkForceRebuildCache()
+    {
+        // GIVEN
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+
+        // WHEN
+        qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8());
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
+        qputenv("QT3D_REBUILD_SHADER_CACHE", "1");
+        Qt3DRender::Render::ShaderBuilder b;
+        b.setGraphicsApi(gl3Api);
+        const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+        b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN -> We have rebuilt the shader file (even if we don't use it)
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+    }
 };
 
 QTEST_MAIN(tst_ShaderBuilder)