Add caching mechanism to ShaderBuilder

It is a 2 level cache system: runtime and offline

First we now generate a hash key based on unique features of a graph,
the api it targets, the last time the graph file was modified.

We then use that hash key to at runtime check whether we have already
loaded a graph matching the same hash key.

If that's not the case, we switch to using the offline cache and checking
whether a file already exists for that shader or not. If the file exists,
we load it, use the code it contains and add it to the runtime cache.

If the file does not exist, we revert to actually generating the shader
and we save the generated code to both the offline cache and the runtime cache.

By default, the offline cache with try to write into the location reported by
QStandardLocation::writableLocation(QStandardLocation::TempLocation).

Optionally, the environment variable QT3D_WRITABLE_CACHE_PATH can be set with
a path to a writable location.

If QT3D_REBUILD_SHADER_CACHE is not empty, cache will be regenerated.
Alternatively if QT3D_DISABLE_SHADER_CACHE is set, cache will be ignored.

Pick-to: 5.15
Change-Id: Ia348f92ce4cdd5e63ec89e58b7954d1f127f26bb
Reviewed-by: Mike Krus <mike.krus@kdab.com>

1 files changed, 199 insertions, 2 deletions

diff --git a/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp b/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp
index 362315d69..9ce63b406 100644
--- a/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp
+++ b/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp
@@ -271,6 +271,7 @@ private slots:
     void shouldHandleEnabledLayersPropertyChange()
     {
         // GIVEN
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
         Qt3DRender::Render::ShaderBuilder backend;
         Qt3DRender::QShaderProgramBuilder frontend;
         TestRenderer renderer;
@@ -468,6 +469,7 @@ private slots:
     void shouldHandleShaderCodeGeneration()
     {
         // GIVEN
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
         Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
         QVERIFY(!Qt3DRender::Render::ShaderBuilder::getPrototypeNames().isEmpty());
 
@@ -555,8 +557,7 @@ private slots:
     void checkCodeUpdatedNotification()
     {
         // GIVEN
-        QSKIP("Disabled for Qt Base QShaderGenerator Integration");
-
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
         Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
         QVERIFY(!Qt3DRender::Render::ShaderBuilder::getPrototypeNames().isEmpty());
         QFETCH(Qt3DRender::QShaderProgram::ShaderType, type);
@@ -604,6 +605,202 @@ private slots:
         QVERIFY(!backend.isShaderCodeDirty(type));
         QCOMPARE(backend.shaderCode(type), gl3Code);
     }
+
+    void checkFileCaching()
+    {
+        // GIVEN
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        const auto gl2Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::NoProfile;
+            api.m_major = 2;
+            api.m_minor = 0;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+        qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8());
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+        QByteArray hashKey;
+        {
+            // WHEN
+            Qt3DRender::Render::ShaderBuilder b;
+            b.setGraphicsApi(gl3Api);
+            const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+            b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+            b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+            // THEN
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+
+            hashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex);
+            QCOMPARE(hashKey.length(), 40);
+
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(),
+                     QString::fromUtf8(hashKey) + QLatin1String(".qt3d"));
+        }
+        {
+            // WHEN
+            Qt3DRender::Render::ShaderBuilder b;
+            b.setGraphicsApi(gl3Api);
+            const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+            b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+            b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+            // THEN
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(),
+                     QString::fromUtf8(hashKey) + QLatin1String(".qt3d"));
+        }
+        {
+            // WHEN
+            Qt3DRender::Render::ShaderBuilder b;
+            b.setGraphicsApi(gl2Api);
+            const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+            b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+            b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+            QByteArray gl2HashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex);
+
+            // THEN
+            QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 2);
+            QVERIFY(gl2HashKey != hashKey);
+        }
+    }
+
+    void checkRuntimeCaching()
+    {
+        // GIVEN
+        TestRenderer renderer;
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+        qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8());
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+
+        // WHEN
+        Qt3DRender::Render::ShaderBuilder b;
+        b.setGraphicsApi(gl3Api);
+        const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+        b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+
+        const QByteArray hashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex);
+        QCOMPARE(hashKey.length(), 40);
+
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(),
+                 QString::fromUtf8(hashKey) + QLatin1String(".qt3d"));
+
+        QVERIFY(!renderer.containsGeneratedShaderGraph(hashKey));
+
+        // WHEN
+        b.setRenderer(&renderer);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN
+        QVERIFY(renderer.containsGeneratedShaderGraph(hashKey));
+    }
+
+    void checkDontUseCache()
+    {
+        // GIVEN
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+
+        // WHEN
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
+        Qt3DRender::Render::ShaderBuilder b;
+        b.setGraphicsApi(gl3Api);
+        const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+        b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 0);
+    }
+
+    void checkForceRebuildCache()
+    {
+        // GIVEN
+        QTemporaryDir cacheDir;
+
+        if (!cacheDir.isValid()) {
+            QSKIP("Unable to generate cache dir, skipping");
+            return;
+        }
+        const auto gl3Api = []{
+            auto api = Qt3DRender::GraphicsApiFilterData();
+            api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL;
+            api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile;
+            api.m_major = 3;
+            api.m_minor = 2;
+            return api;
+        }();
+        Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json");
+
+        // THEN
+        QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty());
+
+        // WHEN
+        qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8());
+        qputenv("QT3D_DISABLE_SHADER_CACHE", "1");
+        qputenv("QT3D_REBUILD_SHADER_CACHE", "1");
+        Qt3DRender::Render::ShaderBuilder b;
+        b.setGraphicsApi(gl3Api);
+        const auto graphUrl = QUrl::fromEncoded("qrc:/input.json");
+        b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl);
+        b.generateCode(Qt3DRender::QShaderProgram::Vertex);
+
+        // THEN -> We have rebuilt the shader file (even if we don't use it)
+        QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1);
+    }
 };
 
 QTEST_MAIN(tst_ShaderBuilder)