Shader caching improvements

Added shader cache load error generation to some cases where it was
missing.

Properly pass the error to surface viewer.

If there was a cache load error, allow exporting the generated cache.

Added overloads to generate shader cache with specified compression
level.

Change-Id: I06d56114918ada23d46474e9466f6031ad8037e2
Fixes: QT3DS-4071
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>
Reviewed-by: Janne Kangas <janne.kangas@qt.io>
Reviewed-by: Antti Määttä <antti.maatta@qt.io>

7 files changed, 170 insertions, 19 deletions

diff --git a/src/api/studio3d/q3dspresentation.cpp b/src/api/studio3d/q3dspresentation.cpp
index 31243d0..f0ca867 100644
--- a/src/api/studio3d/q3dspresentation.cpp
+++ b/src/api/studio3d/q3dspresentation.cpp
@@ -649,11 +649,7 @@ void Q3DSPresentation::unloadSlide(const QString &elementPath)
  */
 void Q3DSPresentation::exportShaderCache(const QUrl &shaderCacheFile, bool binaryShaders)
 {
-    d_ptr->exportShaderCache(binaryShaders, false);
-    if (d_ptr->m_commandQueue)
-        d_ptr->m_shaderCacheWritePending = shaderCacheFile;
-    else
-        d_ptr->writeShaderCache(shaderCacheFile);
+    exportShaderCache(shaderCacheFile, binaryShaders, -1);
 }
 
 /*!
@@ -705,7 +701,7 @@ void Q3DSPresentation::exportShaderCache(const QUrl &shaderCacheFile, bool binar
  */
 void Q3DSPresentation::exportShaderCache(bool binaryShaders)
 {
-    d_ptr->exportShaderCache(binaryShaders, true);
+    exportShaderCache(binaryShaders, -1);
 }
 
 /*!
@@ -726,7 +722,138 @@ void Q3DSPresentation::exportShaderCache(bool binaryShaders)
  */
 void Q3DSPresentation::exportShaderCache(QByteArray &cacheData, bool binaryShaders)
 {
-    d_ptr->exportShaderCache(binaryShaders, false);
+    exportShaderCache(cacheData, binaryShaders, -1);
+}
+
+/*!
+    \qmlmethod Presentation::exportShaderCache(url shaderCacheFile, bool binaryShaders, int compressionLevel)
+    \since QtStudio3D.OpenGL 2.7
+
+    Writes the shaders currently in use to the file specified by \a shaderCacheFile URL.
+    If \a binaryShaders property is \c{true}, precompiled shaders are exported. Otherwise,
+    compressed shader source code is exported.
+    The exported shaders are compressed using compression level specified by \a{compressionLevel}.
+    \c{-1} means default qCompress compression level. \c{0} means no compression.
+
+    Exporting shader cache is an asynchronous operation.
+    The shaderCacheExported signal is emitted when the export is complete.
+
+    Exporting shader cache should be done at the point of application execution where all the
+    shaders that should be initialized at application startup have been initialized.
+
+    \note Exporting shader cache is only supported if no shaders have been originally loaded
+    from a shader cache. Specifying no shader cache file or an empty or invalid shader cache file
+    with shaderCacheFile property allows shader generation.
+
+    \sa shaderCacheFile, shaderCacheExported, qCompress
+ */
+/*!
+    \since Qt 3D Studio 2.7
+    Writes the shaders currently in use to the file specified by \a shaderCacheFile URL.
+    If \a binaryShaders property is \c{true}, precompiled shaders are exported. Otherwise,
+    compressed shader source code is exported.
+    The exported shaders are compressed using compression level specified by \a{compressionLevel}.
+    \c{-1} means default qCompress compression level. \c{0} means no compression.
+
+    The shaderCacheExported signal is emitted when the export is complete.
+
+    Exporting shader cache should be done at the point of application execution where all the
+    shaders that should be initialized at application startup have been initialized.
+
+    \note Exporting shader cache is only supported if no shaders have been originally loaded
+    from a shader cache. Specifying no shader cache file or an empty or invalid shader cache file
+    with shaderCacheFile property allows shader generation.
+
+    \sa shaderCacheFile, shaderCacheExported, qCompress
+ */
+void Q3DSPresentation::exportShaderCache(const QUrl &shaderCacheFile, bool binaryShaders,
+                                         int compressionLevel)
+{
+    d_ptr->exportShaderCache(binaryShaders, false, compressionLevel);
+    if (d_ptr->m_commandQueue)
+        d_ptr->m_shaderCacheWritePending = shaderCacheFile;
+    else
+        d_ptr->writeShaderCache(shaderCacheFile);
+}
+
+/*!
+    \qmlmethod Presentation::exportShaderCache(bool binaryShaders, int compressionLevel)
+    \since QtStudio3D.OpenGL 2.7
+
+    Exports the shaders currently in use and dumps the resulting cache encoded with base64 into
+    stderr. This function is provided as a means to extract the shader cache from environments
+    without a writable disk. The base64 output needs to be converted back to binary
+    representation to be usable as a shader cache file. The Qt 3D Studio Viewer provides
+    a command line parameter \c --convert-shader-cache to do this conversion.
+
+    If \a binaryShaders property is \c{true}, precompiled shaders are exported.
+    Otherwise, compressed shader source code is exported.
+    The exported shaders are compressed using compression level specified by \a{compressionLevel}.
+    \c{-1} means default qCompress compression level. \c{0} means no compression.
+
+    Exporting shader cache is an asynchronous operation.
+    The shaderCacheExported signal is emitted when the export is complete.
+
+    Exporting shader cache should be done at the point of application execution where all the
+    shaders that should be initialized at application startup have been initialized.
+
+    \note Exporting shader cache is only supported if no shaders have been originally loaded
+    from a shader cache. Specifying no shader cache file or an empty or invalid shader cache file
+    with shaderCacheFile property allows shader generation.
+
+    \sa shaderCacheFile, shaderCacheExported, qCompress
+ */
+/*!
+    \since Qt 3D Studio 2.7
+    Exports the shaders currently in use and dumps the resulting cache encoded with base64 into
+    stderr. This function is provided as a means to extract the shader cache from environments
+    without a writable disk. The base64 output needs to be converted back to binary
+    representation to be usable as a shader cache file. The Qt 3D Studio Viewer provides
+    a command line parameter \c --convert-shader-cache to do this conversion.
+
+    If \a binaryShaders property is \c{true}, precompiled shaders are exported.
+    Otherwise, compressed shader source code is exported.
+    The exported shaders are compressed using compression level specified by \a{compressionLevel}.
+    \c{-1} means default qCompress compression level. \c{0} means no compression.
+
+    The shaderCacheExported signal is emitted when the export is complete.
+
+    Exporting shader cache should be done at the point of application execution where all the
+    shaders that should be initialized at application startup have been initialized.
+
+    \note Exporting shader cache is only supported if no shaders have been originally loaded
+    from a shader cache. Specifying no shader cache file or an empty or invalid shader cache file
+    with shaderCacheFile property allows shader generation.
+
+    \sa shaderCacheFile, shaderCacheExported, qCompress
+ */
+void Q3DSPresentation::exportShaderCache(bool binaryShaders, int compressionLevel)
+{
+    d_ptr->exportShaderCache(binaryShaders, true, compressionLevel);
+}
+
+/*!
+    \since Qt 3D Studio 2.7
+    Exports the shaders currently in use to a byte array specified by \a cacheData parameter.
+    If \a binaryShaders property is \c{true}, precompiled shaders are exported. Otherwise,
+    compressed shader source code is exported.
+    The exported shaders are compressed using compression level specified by \a{compressionLevel}.
+    \c{-1} means default qCompress compression level. \c{0} means no compression.
+
+    The shaderCacheExported signal is emitted when the export is complete.
+
+    Exporting shader cache should be done at the point of application execution where all the
+    shaders that should be initialized at application startup have been initialized.
+
+    \note Exporting shader cache is only supported if no shaders have been originally loaded
+    from a shader cache.
+
+    \sa setShaderCacheData, shaderCacheExported, qCompress
+ */
+void Q3DSPresentation::exportShaderCache(QByteArray &cacheData, bool binaryShaders,
+                                         int compressionLevel)
+{
+    d_ptr->exportShaderCache(binaryShaders, false, compressionLevel);
     d_ptr->getShaderCacheData(cacheData);
 }
 
@@ -1970,7 +2097,7 @@ void Q3DSPresentationPrivate::requestResponseHandler(CommandType commandType, vo
         if (response->size() > 0)
             m_shaderCacheExport = response->at(0).toByteArray();
         if (!m_shaderCacheExport.isEmpty())
-            m_shaderCacheExport = qCompress(m_shaderCacheExport);
+            m_shaderCacheExport = qCompress(m_shaderCacheExport, m_shaderCacheCompression);
         if (!m_shaderCacheWritePending.isEmpty()) {
             writeShaderCache(m_shaderCacheWritePending);
             m_shaderCacheWritePending.clear();
@@ -2022,31 +2149,43 @@ void Q3DSPresentationPrivate::getShaderCacheData(QByteArray &shaderCacheData)
 
 QByteArray Q3DSPresentationPrivate::loadShaderCache() const
 {
+    QString error;
     if (!m_shaderCacheFile.isEmpty()) {
         QFile file(Q3DSUtils::urlToLocalFileOrQrc(m_shaderCacheFile));
-        if (file.open(QIODevice::ReadOnly))
-            return qUncompress(file.readAll());
-
-        qWarning() << __FUNCTION__ << "Warning: Failed to read shader cache:"
-                   << m_shaderCacheFile << file.errorString();
+        if (file.open(QIODevice::ReadOnly)) {
+            const QByteArray data = qUncompress(file.readAll());
+            if (!data.isEmpty())
+                return data;
+        }
+        error = QStringLiteral("Failed to read or uncompress shader cache: %1 '%2'")
+                .arg(m_shaderCacheFile.toString()).arg(file.errorString());
     } else if (!m_shaderCacheImport.isEmpty()) {
-        return qUncompress(m_shaderCacheImport);
+        const QByteArray data = qUncompress(m_shaderCacheImport);
+        if (data.isEmpty())
+            error = QStringLiteral("Failed uncompress shader cache.");
+        else
+            return data;
     }
 
+    if (!error.isEmpty())
+        Q_EMIT q_ptr->shaderCacheLoadErrors(error);
+
     return {};
 }
 
-void Q3DSPresentationPrivate::exportShaderCache(bool binaryShaders, bool dumpCache)
+void Q3DSPresentationPrivate::exportShaderCache(bool binaryShaders, bool dumpCache,
+                                                int compression)
 {
     if (m_viewerApp) {
         m_shaderCacheExport = m_viewerApp->exportShaderCache(binaryShaders);
         if (!m_shaderCacheExport.isEmpty())
-            m_shaderCacheExport = qCompress(m_shaderCacheExport);
+            m_shaderCacheExport = qCompress(m_shaderCacheExport, compression);
         if (dumpCache)
             dumpShaderCache();
     } else if (m_commandQueue) {
         m_commandQueue->queueCommand(CommandType_RequestExportShaderCache, binaryShaders);
         m_shaderCacheDumpPending = dumpCache;
+        m_shaderCacheCompression = compression;
     }
 }
 
diff --git a/src/api/studio3d/q3dspresentation.h b/src/api/studio3d/q3dspresentation.h
index 44bcad5..6d25a74 100644
--- a/src/api/studio3d/q3dspresentation.h
+++ b/src/api/studio3d/q3dspresentation.h
@@ -97,6 +97,10 @@ public:
                                                      bool binaryShaders);
     Q_REVISION(1) Q_INVOKABLE void exportShaderCache(bool binaryShaders);
     void exportShaderCache(QByteArray &cacheData, bool binaryShaders);
+    Q_REVISION(2) Q_INVOKABLE void exportShaderCache(const QUrl &shaderCacheFile,
+                                                     bool binaryShaders, int compressionLevel);
+    Q_REVISION(2) Q_INVOKABLE void exportShaderCache(bool binaryShaders, int compressionLevel);
+    void exportShaderCache(QByteArray &cacheData, bool binaryShaders, int compressionLevel);
     void setShaderCacheData(const QByteArray &shaderCache);
     QUrl shaderCacheFile() const;
 
diff --git a/src/api/studio3d/q3dspresentation_p.h b/src/api/studio3d/q3dspresentation_p.h
index e431a30..a8379fa 100644
--- a/src/api/studio3d/q3dspresentation_p.h
+++ b/src/api/studio3d/q3dspresentation_p.h
@@ -104,7 +104,7 @@ public:
     void writeShaderCache(const QUrl &shaderCacheFile);
     void getShaderCacheData(QByteArray &shaderCacheData);
     QByteArray loadShaderCache() const;
-    void exportShaderCache(bool binaryShaders, bool dumpCache);
+    void exportShaderCache(bool binaryShaders, bool dumpCache, int compression);
     void dumpShaderCache();
 
 public Q_SLOTS:
@@ -136,6 +136,7 @@ private:
     QByteArray m_shaderCacheImport;
     QUrl m_shaderCacheWritePending;
     bool m_shaderCacheDumpPending = false;
+    bool m_shaderCacheCompression = -1;
     int m_dataInputCallIndex = 0;
 
     friend class Q3DSStudio3D;
diff --git a/src/api/studio3d/q3dssurfaceviewer.cpp b/src/api/studio3d/q3dssurfaceviewer.cpp
index e541e02..1e1a7ed 100644
--- a/src/api/studio3d/q3dssurfaceviewer.cpp
+++ b/src/api/studio3d/q3dssurfaceviewer.cpp
@@ -657,6 +657,8 @@ bool Q3DSSurfaceViewerPrivate::initializeRuntime()
                 this->q_ptr, &Q3DSSurfaceViewer::presentationReady);
         connect(m_viewerApp, &Q3DSViewerApp::SigPresentationLoaded,
                 this->q_ptr, &Q3DSSurfaceViewer::presentationLoaded);
+        connect(m_viewerApp, &Q3DSViewerApp::SigLoadShaderCacheErrors,
+                m_presentation, &Q3DSPresentation::shaderCacheLoadErrors);
         Q_ASSERT(m_viewerApp);
     }
     if (!m_context->makeCurrent(m_surface)) {
diff --git a/src/api/studio3dqml/q3dsplugin.cpp b/src/api/studio3dqml/q3dsplugin.cpp
index f481ed3..ee9a9b9 100644
--- a/src/api/studio3dqml/q3dsplugin.cpp
+++ b/src/api/studio3dqml/q3dsplugin.cpp
@@ -63,6 +63,11 @@ void Q3DSPlugin::registerTypes(const char *uri)
     qmlRegisterType<Q3DSStudio3D, 1>(uri, 2, 5, "Studio3D");
     qmlRegisterType<Q3DSViewerSettings, 1>(uri, 2, 5, "ViewerSettings");
     qmlRegisterType<Q3DSPresentationItem, 1>(uri, 2, 5, "Presentation");
+    qmlRegisterRevision<Q3DSPresentation, 1>(uri, 2, 5);
+
+    // 2.7
+    qmlRegisterType<Q3DSPresentationItem, 2>(uri, 2, 7, "Presentation");
+    qmlRegisterRevision<Q3DSPresentation, 2>(uri, 2, 7);
 
     // Automatically register the latest version
     qmlRegisterModule(uri, ((QTSTUDIO3D_VERSION >> 16) & 0xff), ((QTSTUDIO3D_VERSION >> 8) & 0xff));
diff --git a/src/render/Qt3DSRenderShaderProgram.cpp b/src/render/Qt3DSRenderShaderProgram.cpp
index b5f2a77..2ed881a 100644
--- a/src/render/Qt3DSRenderShaderProgram.cpp
+++ b/src/render/Qt3DSRenderShaderProgram.cpp
@@ -1222,9 +1222,8 @@ namespace render {
         bProgramIsValid = pProgram->link(format, &binary);
 
         if (!bProgramIsValid && pProgram) {
-            NVFoundationBase &foundation(context.GetFoundation());
             qCCritical(INTERNAL_ERROR, "Failed to link binary program!!");
-            WriteErrorMessage(foundation, "Program link output:", pProgram->GetErrorMessage());
+            result.errors = QStringLiteral("Binary program link failed");
 
             // delete program
             QT3DS_FREE(context.GetFoundation().getAllocator(), pProgram);
diff --git a/src/runtimerender/Qt3DSRenderShaderCache.cpp b/src/runtimerender/Qt3DSRenderShaderCache.cpp
index 3794a83..b1a8c4b 100644
--- a/src/runtimerender/Qt3DSRenderShaderCache.cpp
+++ b/src/runtimerender/Qt3DSRenderShaderCache.cpp
@@ -678,6 +678,7 @@ struct ShaderCache : public IShaderCache
             if (!theShader) {
                 qWarning() << __FUNCTION__ << "Failed to load a cached a shader:" << key;
                 m_Shaders.erase(theKey);
+                m_shadersInitializedFromCache = false;
             }
         }
     }