diff options
| -rw-r--r-- | src/render/backend/abstractrenderer_p.h | 9 | ||||
| -rw-r--r-- | src/render/materialsystem/shaderbuilder.cpp | 106 | ||||
| -rw-r--r-- | src/render/materialsystem/shaderbuilder_p.h | 2 | ||||
| -rw-r--r-- | tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp | 201 |
4 files changed, 311 insertions, 7 deletions
diff --git a/src/render/backend/abstractrenderer_p.h b/src/render/backend/abstractrenderer_p.h index 0dd24dcd8..ec08ab641 100644 --- a/src/render/backend/abstractrenderer_p.h +++ b/src/render/backend/abstractrenderer_p.h @@ -190,6 +190,15 @@ public: // These commands are executed in a dedicated command thread // More will be added later virtual void loadShader(Shader *shader, Qt3DRender::Render::HShader shaderHandle) = 0; + + // Runtime Cache for Generated Shader Graph + bool containsGeneratedShaderGraph(const QByteArray &key) const { return m_cachedGeneratedShaderGraphes.contains(key); }; + QByteArray cachedGeneratedShaderGraph(const QByteArray &key) const { return m_cachedGeneratedShaderGraphes.value(key); }; + void insertGeneratedShaderGraph(const QByteArray &key, const QByteArray shaderCode) { m_cachedGeneratedShaderGraphes.insert(key, shaderCode); } + void removeGeneratedShaderGraph(const QByteArray &key) { m_cachedGeneratedShaderGraphes.remove(key); }; + +private: + QHash<QByteArray, QByteArray> m_cachedGeneratedShaderGraphes; }; Q_DECLARE_OPERATORS_FOR_FLAGS(AbstractRenderer::BackendNodeDirtySet) diff --git a/src/render/materialsystem/shaderbuilder.cpp b/src/render/materialsystem/shaderbuilder.cpp index b168756c9..d7ccea7da 100644 --- a/src/render/materialsystem/shaderbuilder.cpp +++ b/src/render/materialsystem/shaderbuilder.cpp @@ -54,6 +54,10 @@ #include <QFile> #include <QFileInfo> #include <QUrl> +#include <QCryptographicHash> +#include <QDateTime> +#include <QStandardPaths> +#include <QDir> static void initResources() { @@ -222,6 +226,53 @@ void ShaderBuilder::generateCode(QShaderProgram::ShaderType type) return; } + auto updateShaderCodeAndClearDirty = [&] (const QByteArray &shaderCode) { + m_codes.insert(type, shaderCode); + m_dirtyTypes.remove(type); + m_pendingUpdates.push_back({ peerId(), + type, + m_codes.value(type) }); + }; + + const QByteArray cacheKey = hashKeyForShaderGraph(type); + const bool forceRegenerate = qEnvironmentVariableIsSet("QT3D_REBUILD_SHADER_CACHE"); + const bool useCache = !qEnvironmentVariableIsSet("QT3D_DISABLE_SHADER_CACHE") || !forceRegenerate; + const QByteArray userProvidedPath = qgetenv("QT3D_WRITABLE_CACHE_PATH"); + const QString cachedFilterPath = QDir(userProvidedPath.isEmpty() ? + QStandardPaths::writableLocation(QStandardPaths::TempLocation) + : QString::fromUtf8(userProvidedPath)).absoluteFilePath(QString::fromUtf8(cacheKey) + QLatin1String(".qt3d")); + QFile cachedShaderFile(cachedFilterPath); + + // Check our runtime cache to see if we have already loaded the shader previously + if (useCache) { + // We check if we already have generated a shader previously for the + // given type, the given graph, the given API and the current set of layer + // If that's the case it's faster to load the pre generated shader file + + if (m_renderer && m_renderer->containsGeneratedShaderGraph(cacheKey)) { + updateShaderCodeAndClearDirty(m_renderer->cachedGeneratedShaderGraph(cacheKey)); + return; + } + + // else check if a cachedShader file exists + if (cachedShaderFile.exists()) { + if (!cachedShaderFile.open(QFile::ReadOnly)) { + qWarning() << "Couldn't open cached shader file:" << graphPath; + // Too bad, we have to generate the shader below + } else { + // Use cached shader + const QByteArray shaderCode = cachedShaderFile.readAll(); + updateShaderCodeAndClearDirty(shaderCode); + + // Record to runtime cache + if (m_renderer) + m_renderer->insertGeneratedShaderGraph(cacheKey, shaderCode); + return; + } + } + } + + // Generate Shader and Cache the result for subsequent uses auto graphLoader = QShaderGraphLoader(); graphLoader.setPrototypes(qt3dGlobalShaderPrototypes->prototypes()); graphLoader.setDevice(&file); @@ -249,12 +300,20 @@ void ShaderBuilder::generateCode(QShaderProgram::ShaderType type) const auto code = generator.createShaderCode(m_enabledLayers); const auto deincludified = QShaderProgramPrivate::deincludify(code, graphPath + QStringLiteral(".glsl")); - m_codes.insert(type, deincludified); - m_dirtyTypes.remove(type); - m_pendingUpdates.push_back({ peerId(), - type, - m_codes.value(type) }); + updateShaderCodeAndClearDirty(deincludified); + + // Record to runtime cache + if (useCache || forceRegenerate) { + if (m_renderer) + m_renderer->insertGeneratedShaderGraph(cacheKey, deincludified); + + // Record to file cache + if (cachedShaderFile.open(QFile::WriteOnly)) + cachedShaderFile.write(deincludified); + else + qWarning() << "Unable to write cached shader file"; + } } void ShaderBuilder::syncFromFrontEnd(const QNode *frontEnd, bool firstTime) @@ -299,6 +358,43 @@ void ShaderBuilder::syncFromFrontEnd(const QNode *frontEnd, bool firstTime) } } +QByteArray ShaderBuilder::hashKeyForShaderGraph(QShaderProgram::ShaderType type) const +{ + const auto graphPath = Qt3DRender::QUrlHelper::urlToLocalFileOrQrc(shaderGraph(type)); + QFile file(graphPath); + if (!file.exists()) { + qWarning() << graphPath << "doesn't exist"; + return {}; + } + + QCryptographicHash hashBuilder(QCryptographicHash::Sha1); + // Add graphPath + hashBuilder.addData(graphPath.toUtf8()); + // Get TimeStamp and Graph file size + QFileInfo info(graphPath); + const QString fileInfo = QString::fromUtf8("%1_%2") + .arg(info.lastModified().toSecsSinceEpoch()) + .arg(info.size()); + hashBuilder.addData(fileInfo.toUtf8()); + + // Add Layers + for (const QString &layer : m_enabledLayers) + hashBuilder.addData(layer.toUtf8()); + + // Add GraphicsInfo + const QString graphicsInfo = QString::fromUtf8("API: %1 Profile: %2 Major: %3 Minor: %4") + .arg(int(m_graphicsApi.m_api)) + .arg(int(m_graphicsApi.m_profile)) + .arg(int(m_graphicsApi.m_major)) + .arg(int(m_graphicsApi.m_minor)); + hashBuilder.addData(graphicsInfo.toUtf8()); + + // Add Shader Type + hashBuilder.addData(QString::number(type).toUtf8()); + + return hashBuilder.result().toHex(); +} + } // namespace Render } // namespace Qt3DRender diff --git a/src/render/materialsystem/shaderbuilder_p.h b/src/render/materialsystem/shaderbuilder_p.h index 0a799afaa..2aaad882b 100644 --- a/src/render/materialsystem/shaderbuilder_p.h +++ b/src/render/materialsystem/shaderbuilder_p.h @@ -97,6 +97,8 @@ public: QVector<ShaderBuilderUpdate> takePendingUpdates() { return std::move(m_pendingUpdates); } + QByteArray hashKeyForShaderGraph(QShaderProgram::ShaderType type) const; + private: void setEnabledLayers(const QStringList &layers); diff --git a/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp b/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp index 9be7db5f1..2bab9d6f5 100644 --- a/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp +++ b/tests/auto/render/shaderbuilder/tst_shaderbuilder.cpp @@ -272,6 +272,7 @@ private slots: void shouldHandleEnabledLayersPropertyChange() { // GIVEN + qputenv("QT3D_DISABLE_SHADER_CACHE", "1"); Qt3DRender::Render::ShaderBuilder backend; Qt3DRender::QShaderProgramBuilder frontend; TestRenderer renderer; @@ -469,6 +470,7 @@ private slots: void shouldHandleShaderCodeGeneration() { // GIVEN + qputenv("QT3D_DISABLE_SHADER_CACHE", "1"); Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json"); QVERIFY(!Qt3DRender::Render::ShaderBuilder::getPrototypeNames().isEmpty()); @@ -556,8 +558,7 @@ private slots: void checkCodeUpdatedNotification() { // GIVEN - QSKIP("Disabled for Qt Base QShaderGenerator Integration"); - + qputenv("QT3D_DISABLE_SHADER_CACHE", "1"); Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json"); QVERIFY(!Qt3DRender::Render::ShaderBuilder::getPrototypeNames().isEmpty()); QFETCH(Qt3DRender::QShaderProgram::ShaderType, type); @@ -606,6 +607,202 @@ private slots: QVERIFY(!backend.isShaderCodeDirty(type)); QCOMPARE(backend.shaderCode(type), gl3Code); } + + void checkFileCaching() + { + // GIVEN + QTemporaryDir cacheDir; + + if (!cacheDir.isValid()) { + QSKIP("Unable to generate cache dir, skipping"); + return; + } + const auto gl3Api = []{ + auto api = Qt3DRender::GraphicsApiFilterData(); + api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL; + api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile; + api.m_major = 3; + api.m_minor = 2; + return api; + }(); + const auto gl2Api = []{ + auto api = Qt3DRender::GraphicsApiFilterData(); + api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL; + api.m_profile = Qt3DRender::QGraphicsApiFilter::NoProfile; + api.m_major = 2; + api.m_minor = 0; + return api; + }(); + Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json"); + qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8()); + + // THEN + QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty()); + QByteArray hashKey; + { + // WHEN + Qt3DRender::Render::ShaderBuilder b; + b.setGraphicsApi(gl3Api); + const auto graphUrl = QUrl::fromEncoded("qrc:/input.json"); + b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + + // THEN + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1); + + hashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex); + QCOMPARE(hashKey.length(), 40); + + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(), + QString::fromUtf8(hashKey) + QLatin1String(".qt3d")); + } + { + // WHEN + Qt3DRender::Render::ShaderBuilder b; + b.setGraphicsApi(gl3Api); + const auto graphUrl = QUrl::fromEncoded("qrc:/input.json"); + b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + + // THEN + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1); + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(), + QString::fromUtf8(hashKey) + QLatin1String(".qt3d")); + } + { + // WHEN + Qt3DRender::Render::ShaderBuilder b; + b.setGraphicsApi(gl2Api); + const auto graphUrl = QUrl::fromEncoded("qrc:/input.json"); + b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + QByteArray gl2HashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex); + + // THEN + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 2); + QVERIFY(gl2HashKey != hashKey); + } + } + + void checkRuntimeCaching() + { + // GIVEN + TestRenderer renderer; + QTemporaryDir cacheDir; + + if (!cacheDir.isValid()) { + QSKIP("Unable to generate cache dir, skipping"); + return; + } + const auto gl3Api = []{ + auto api = Qt3DRender::GraphicsApiFilterData(); + api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL; + api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile; + api.m_major = 3; + api.m_minor = 2; + return api; + }(); + Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json"); + qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8()); + + // THEN + QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty()); + + // WHEN + Qt3DRender::Render::ShaderBuilder b; + b.setGraphicsApi(gl3Api); + const auto graphUrl = QUrl::fromEncoded("qrc:/input.json"); + b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + + // THEN + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1); + + const QByteArray hashKey = b.hashKeyForShaderGraph(Qt3DRender::QShaderProgram::Vertex); + QCOMPARE(hashKey.length(), 40); + + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).first(), + QString::fromUtf8(hashKey) + QLatin1String(".qt3d")); + + QVERIFY(!renderer.containsGeneratedShaderGraph(hashKey)); + + // WHEN + b.setRenderer(&renderer); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + + // THEN + QVERIFY(renderer.containsGeneratedShaderGraph(hashKey)); + } + + void checkDontUseCache() + { + // GIVEN + QTemporaryDir cacheDir; + + if (!cacheDir.isValid()) { + QSKIP("Unable to generate cache dir, skipping"); + return; + } + const auto gl3Api = []{ + auto api = Qt3DRender::GraphicsApiFilterData(); + api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL; + api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile; + api.m_major = 3; + api.m_minor = 2; + return api; + }(); + Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json"); + + // THEN + QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty()); + + // WHEN + qputenv("QT3D_DISABLE_SHADER_CACHE", "1"); + Qt3DRender::Render::ShaderBuilder b; + b.setGraphicsApi(gl3Api); + const auto graphUrl = QUrl::fromEncoded("qrc:/input.json"); + b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + + // THEN + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 0); + } + + void checkForceRebuildCache() + { + // GIVEN + QTemporaryDir cacheDir; + + if (!cacheDir.isValid()) { + QSKIP("Unable to generate cache dir, skipping"); + return; + } + const auto gl3Api = []{ + auto api = Qt3DRender::GraphicsApiFilterData(); + api.m_api = Qt3DRender::QGraphicsApiFilter::OpenGL; + api.m_profile = Qt3DRender::QGraphicsApiFilter::CoreProfile; + api.m_major = 3; + api.m_minor = 2; + return api; + }(); + Qt3DRender::Render::ShaderBuilder::setPrototypesFile(":/prototypes.json"); + + // THEN + QVERIFY(QDir(cacheDir.path()).entryList(QDir::Files).empty()); + + // WHEN + qputenv("QT3D_WRITABLE_CACHE_PATH", cacheDir.path().toUtf8()); + qputenv("QT3D_DISABLE_SHADER_CACHE", "1"); + qputenv("QT3D_REBUILD_SHADER_CACHE", "1"); + Qt3DRender::Render::ShaderBuilder b; + b.setGraphicsApi(gl3Api); + const auto graphUrl = QUrl::fromEncoded("qrc:/input.json"); + b.setShaderGraph(Qt3DRender::QShaderProgram::Vertex, graphUrl); + b.generateCode(Qt3DRender::QShaderProgram::Vertex); + + // THEN -> We have rebuilt the shader file (even if we don't use it) + QCOMPARE(QDir(cacheDir.path()).entryList(QDir::Files).count(), 1); + } }; QTEST_MAIN(tst_ShaderBuilder) |