Indicate effect shader errors in Editorv2.5.0-beta2

Show a pop-up if a shader fails to compile. Compilation takes place when
the effect is added to a object in timeline.

Requires both editor and runtime commits.

Task-id: QT3DS-3598
Change-Id: Ie9b5a960773c16e00ef99bd0a05e581daf008365
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>

10 files changed, 63 insertions, 27 deletions

diff --git a/src/render/Qt3DSRenderBaseTypes.h b/src/render/Qt3DSRenderBaseTypes.h
index fa5e6fc..909077d 100644
--- a/src/render/Qt3DSRenderBaseTypes.h
+++ b/src/render/Qt3DSRenderBaseTypes.h
@@ -37,6 +37,7 @@
 #include "foundation/Qt3DSSimpleTypes.h"
 #include "foundation/Qt3DSMath.h"
 #include "foundation/Qt3DSVec2.h"
+#include <QtCore/qstring.h>
 
 namespace qt3ds {
 
@@ -1077,6 +1078,8 @@ struct NVRenderVertFragCompilationResult
 
     NVRenderShaderProgram *mShader; ///< contains the program
 
+    QString errors;
+
     NVRenderVertFragCompilationResult()
         : mShaderName("")
         , mShader(NULL)
diff --git a/src/render/Qt3DSRenderShaderProgram.cpp b/src/render/Qt3DSRenderShaderProgram.cpp
index cc8c1fd..b5f2a77 100644
--- a/src/render/Qt3DSRenderShaderProgram.cpp
+++ b/src/render/Qt3DSRenderShaderProgram.cpp
@@ -1115,20 +1115,23 @@ namespace render {
         if (!vertexValid || !fragValid || !tcValid || !teValid || !geValid || !bProgramIsValid) {
             NVFoundationBase &foundation(context.GetFoundation());
 
+            const char *err;
             if (!vertexValid) {
                 qCCritical(INTERNAL_ERROR, "Failed to generate vertex shader!!");
                 qCCritical(INTERNAL_ERROR, "Vertex source:\n%s",
                     nonNull((const char *)vertShaderSource.begin()));
-                WriteErrorMessage(foundation, "Vertex compilation output:",
-                                  vtxShader.getValue()->GetErrorMessage());
+                err = vtxShader.getValue()->GetErrorMessage();
+                result.errors.append(err);
+                WriteErrorMessage(foundation, "Vertex compilation output:", err);
             }
 
             if (!fragValid) {
                 qCCritical(INTERNAL_ERROR, "Failed to generate fragment shader!!");
                 qCCritical(INTERNAL_ERROR, "Fragment source:\n%s",
                     nonNull((const char *)fragShaderSource.begin()));
-                WriteErrorMessage(foundation, "Fragment compilation output:",
-                                  fragShader.getValue()->GetErrorMessage());
+                err = fragShader.getValue()->GetErrorMessage();
+                result.errors.append(err);
+                WriteErrorMessage(foundation, "Fragment compilation output:", err);
             }
 
             if (!tcValid) {
@@ -1136,8 +1139,9 @@ namespace render {
                     "Failed to generate tessellation control shader!!");
                 qCCritical(INTERNAL_ERROR, "Tessellation control source:\n%s",
                     nonNull((const char *)tessControlShaderSource.begin()));
-                WriteErrorMessage(foundation, "Tessellation control compilation output:",
-                                  tcShader.getValue()->GetErrorMessage());
+                err = tcShader.getValue()->GetErrorMessage();
+                result.errors.append(err);
+                WriteErrorMessage(foundation, "Tessellation control compilation output:", err);
             }
 
             if (!teValid) {
@@ -1145,16 +1149,18 @@ namespace render {
                     "Failed to generate tessellation evaluation shader!!");
                 qCCritical(INTERNAL_ERROR, "Tessellation evaluation source:\n%s",
                     nonNull((const char *)tessEvaluationShaderSource.begin()));
-                WriteErrorMessage(foundation, "Tessellation evaluation compilation output:",
-                                  teShader.getValue()->GetErrorMessage());
+                err = teShader.getValue()->GetErrorMessage();
+                result.errors.append(err);
+                WriteErrorMessage(foundation, "Tessellation evaluation compilation output:", err);
             }
 
             if (!geValid) {
                 qCCritical(INTERNAL_ERROR, "Failed to generate geometry shader!!");
                 qCCritical(INTERNAL_ERROR, "Geometry source:\n%s",
                     nonNull((const char *)geometryShaderSource.begin()));
-                WriteErrorMessage(foundation, "Geometry compilation output:",
-                                  geShader.getValue()->GetErrorMessage());
+                err = geShader.getValue()->GetErrorMessage();
+                result.errors.append(err);
+                WriteErrorMessage(foundation, "Geometry compilation output:", err);
             }
 
             if (!bProgramIsValid && pProgram) {
diff --git a/src/runtimerender/Qt3DSRenderDynamicObjectSystem.cpp b/src/runtimerender/Qt3DSRenderDynamicObjectSystem.cpp
index b3395f3..378b59b 100644
--- a/src/runtimerender/Qt3DSRenderDynamicObjectSystem.cpp
+++ b/src/runtimerender/Qt3DSRenderDynamicObjectSystem.cpp
@@ -1325,6 +1325,7 @@ struct SDynamicObjectSystemImpl : public IDynamicObjectSystem
                                          CRegisteredString inProgramMacroName,
                                          TShaderFeatureSet inFeatureSet,
                                          const dynamic::SDynamicShaderProgramFlags &inFlags,
+                                         QString &errors,
                                          bool inForceCompilation = false)
     {
         m_VertShader.clear();
@@ -1385,7 +1386,7 @@ struct SDynamicObjectSystemImpl : public IDynamicObjectSystem
             return theShaderCache.ForceCompileProgram(theKey, m_VertShader.c_str(),
                                                       m_FragShader.c_str(), NULL, NULL,
                                                       m_GeometryShader.c_str(), theFlags,
-                                                      inFeatureSet, false);
+                                                      inFeatureSet, errors, false);
         }
         return theShaderCache.CompileProgram(theKey, m_VertShader.c_str(), m_FragShader.c_str(),
                                              NULL, NULL, m_GeometryShader.c_str(), theFlags,
@@ -1406,7 +1407,7 @@ struct SDynamicObjectSystemImpl : public IDynamicObjectSystem
                                              CRegisteredString inProgramMacro,
                                              TShaderFeatureSet inFeatureSet,
                                              const SDynamicShaderProgramFlags &inFlags,
-                                             bool inForceCompilation) override
+                                             QString &errors, bool inForceCompilation) override
     {
         eastl::pair<const SShaderMapKey, TShaderAndFlags> theInserter(
             SShaderMapKey(TStrStrPair(inPath, inProgramMacro), inFeatureSet, inFlags.m_TessMode,
@@ -1426,8 +1427,9 @@ struct SDynamicObjectSystemImpl : public IDynamicObjectSystem
                     DoLoadShader(inPath, theShaderBuffer);
                     if (theShaderInfo.m_HasGeomShader)
                         theFlags.SetGeometryShaderEnabled(true);
-                    theProgram = CompileShader(inPath, theShaderBuffer.c_str(), NULL, inProgramMacro,
-                                               inFeatureSet, theFlags, inForceCompilation);
+                    theProgram = CompileShader(
+                                inPath, theShaderBuffer.c_str(), NULL, inProgramMacro,
+                                inFeatureSet, theFlags, errors, inForceCompilation);
                 } else {
                     Qt3DSString theShaderBuffer;
                     const char8_t *shaderVersionStr = "#version 430\n";
@@ -1505,8 +1507,9 @@ struct SDynamicObjectSystemImpl : public IDynamicObjectSystem
                 programBuffer.append(fragmentSource);
                 programBuffer.append("\n#endif");
                 flags.SetGeometryShaderEnabled(true);
+                QString error;
                 theProgram = CompileShader(inPath, programBuffer.c_str(), theShaderBuffer.c_str(),
-                                           theProgramMacro, inFeatureSet, flags);
+                                           theProgramMacro, inFeatureSet, flags, error);
             }
             theInsertResult.first->second = TShaderAndFlags(theProgram, flags);
         }
diff --git a/src/runtimerender/Qt3DSRenderDynamicObjectSystem.h b/src/runtimerender/Qt3DSRenderDynamicObjectSystem.h
index cfe022e..5379df2 100644
--- a/src/runtimerender/Qt3DSRenderDynamicObjectSystem.h
+++ b/src/runtimerender/Qt3DSRenderDynamicObjectSystem.h
@@ -265,7 +265,7 @@ namespace render {
         GetShaderProgram(CRegisteredString inPath, CRegisteredString inProgramMacro,
                          TShaderFeatureSet inFeatureSet,
                          const dynamic::SDynamicShaderProgramFlags &inFlags,
-                         bool inForceCompilation = false) = 0;
+                         QString &errors, bool inForceCompilation = false) = 0;
 
         virtual void GetShaderSource(CRegisteredString inPath, Qt3DSString &outSource) = 0;
 
diff --git a/src/runtimerender/Qt3DSRenderEffectSystem.cpp b/src/runtimerender/Qt3DSRenderEffectSystem.cpp
index 795eeea..5b516b8 100644
--- a/src/runtimerender/Qt3DSRenderEffectSystem.cpp
+++ b/src/runtimerender/Qt3DSRenderEffectSystem.cpp
@@ -1020,7 +1020,8 @@ struct SEffectSystem : public IEffectSystem
         return theBuffer;
     }
 
-    SEffectShader *BindShader(CRegisteredString &inEffectId, const SBindShader &inCommand)
+    SEffectShader *BindShader(CRegisteredString &inEffectId, const SBindShader &inCommand,
+                              QString &errors)
     {
         SEffectClass *theClass = GetEffectClass(inEffectId);
         if (!theClass) {
@@ -1039,7 +1040,7 @@ struct SEffectSystem : public IEffectSystem
             NVRenderShaderProgram *theProgram =
                 m_Context->GetDynamicObjectSystem()
                     .GetShaderProgram(inCommand.m_ShaderPath, inCommand.m_ShaderDefine,
-                                      TShaderFeatureSet(), SDynamicShaderProgramFlags(),
+                                      TShaderFeatureSet(), SDynamicShaderProgramFlags(), errors,
                                       forceCompilation).first;
             if (theProgram)
                 theInsertResult.first->second = QT3DS_NEW(m_Allocator, SEffectShader)(*theProgram);
@@ -1546,6 +1547,7 @@ struct SEffectSystem : public IEffectSystem
             theContext.SetDepthWriteEnabled(false);
 
             QT3DSMat44 theMVP(QT3DSMat44::createIdentity());
+            QString errors;
             NVConstDataRef<dynamic::SCommand *> theCommands =
                 inClass.m_DynamicClass->GetRenderCommands();
             for (QT3DSU32 commandIdx = 0, commandEnd = theCommands.size(); commandIdx < commandEnd;
@@ -1592,8 +1594,10 @@ struct SEffectSystem : public IEffectSystem
                 } break;
                 case CommandTypes::BindShader:
                     theCurrentShader = BindShader(inEffect.m_ClassName,
-                                                  static_cast<const SBindShader &>(theCommand));
-                    break;
+                                                  static_cast<const SBindShader &>(theCommand),
+                                                  errors);
+                    if (!errors.isEmpty())
+                        inEffect.SetError(m_CoreContext.GetStringTable().RegisterStr(errors));
                 case CommandTypes::ApplyInstanceValue:
                     if (theCurrentShader)
                         ApplyInstanceValue(inEffect, inClass, *theCurrentShader->m_Shader,
diff --git a/src/runtimerender/Qt3DSRenderShaderCache.cpp b/src/runtimerender/Qt3DSRenderShaderCache.cpp
index c642360..fd2c669 100644
--- a/src/runtimerender/Qt3DSRenderShaderCache.cpp
+++ b/src/runtimerender/Qt3DSRenderShaderCache.cpp
@@ -383,7 +383,7 @@ struct ShaderCache : public IShaderCache
                         const char8_t *inTessCtrl, const char8_t *inTessEval, const char8_t *inGeom,
                         const SShaderCacheProgramFlags &inFlags,
                         NVConstDataRef<SShaderPreprocessorFeature> inFeatures,
-                        bool separableProgram, bool fromDisk = false) override
+                        QString &errors, bool separableProgram, bool fromDisk = false) override
     {
         if (m_ShaderCompilationEnabled == false)
             return nullptr;
@@ -421,14 +421,15 @@ struct ShaderCache : public IShaderCache
         if (!fromDisk)
             addShaderPreprocessors(inKey, inFlags, inFeatures, separableProgram, false);
 
-        theInserter.first->second =
-            m_RenderContext
+        NVRenderVertFragCompilationResult res = m_RenderContext
                 .CompileSource(inKey, m_VertexCode.c_str(), QT3DSU32(m_VertexCode.size()),
                                m_FragmentCode.c_str(), QT3DSU32(m_FragmentCode.size()),
                                m_TessCtrlCode.c_str(), QT3DSU32(m_TessCtrlCode.size()),
                                m_TessEvalCode.c_str(), QT3DSU32(m_TessEvalCode.size()),
                                m_GeometryCode.c_str(), QT3DSU32(m_GeometryCode.size()),
-                               separableProgram).mShader;
+                               separableProgram);
+        theInserter.first->second = res.mShader;
+        errors = res.errors;
 
         // This is unnecessary memory waste in final deployed product, so we don't store this
         // information when shaders were initialized from a cache.
@@ -462,9 +463,10 @@ struct ShaderCache : public IShaderCache
         if (theProgram)
             return theProgram;
 
+        QString error;
         NVRenderShaderProgram *retval =
             ForceCompileProgram(inKey, inVert, inFrag, inTessCtrl, inTessEval, inGeom, inFlags,
-                                inFeatures, separableProgram);
+                                inFeatures, error, separableProgram);
         return retval;
     }
 
@@ -622,6 +624,7 @@ struct ShaderCache : public IShaderCache
 
                 if (!loadVertexData.isEmpty() && (!loadFragmentData.isEmpty()
                                                   || !loadGeometryData.isEmpty())) {
+                    QString error;
                     theShader = ForceCompileProgram(
                                 theKey, loadVertexData.constData(),
                                 loadFragmentData.constData(),
@@ -631,7 +634,7 @@ struct ShaderCache : public IShaderCache
                                 SShaderCacheProgramFlags(),
                                 qt3ds::foundation::toDataRef(
                                     features.data(), static_cast<QT3DSU32>(features.size())),
-                                false, true);
+                                error, false, true);
                 }
             }
             // If something doesn't save or load correctly, get the runtime to re-generate.
diff --git a/src/runtimerender/Qt3DSRenderShaderCache.h b/src/runtimerender/Qt3DSRenderShaderCache.h
index ca333c6..94080ed 100644
--- a/src/runtimerender/Qt3DSRenderShaderCache.h
+++ b/src/runtimerender/Qt3DSRenderShaderCache.h
@@ -123,7 +123,7 @@ namespace render {
         ForceCompileProgram(CRegisteredString inKey, const char8_t *inVert, const char8_t *inFrag,
                             const char8_t *inTessCtrl, const char8_t *inTessEval,
                             const char8_t *inGeom, const SShaderCacheProgramFlags &inFlags,
-                            TShaderFeatureSet inFeatures, bool separableProgram,
+                            TShaderFeatureSet inFeatures, QString &errors, bool separableProgram,
                             bool fromDisk = false) = 0;
 
         // It is up to the caller to ensure that inFeatures contains unique keys.
diff --git a/src/runtimerender/graphobjects/Qt3DSRenderEffect.cpp b/src/runtimerender/graphobjects/Qt3DSRenderEffect.cpp
index d22c3c3..279946a 100644
--- a/src/runtimerender/graphobjects/Qt3DSRenderEffect.cpp
+++ b/src/runtimerender/graphobjects/Qt3DSRenderEffect.cpp
@@ -42,6 +42,7 @@ void SEffect::Initialize()
     m_NextEffect = NULL;
     m_Context = NULL;
     m_imageMaps = nullptr;
+    m_error = CRegisteredString();
 }
 
 void SEffect::SetActive(bool inActive, IEffectSystem &inManager)
@@ -60,3 +61,13 @@ void SEffect::Reset(IEffectSystem &inSystem)
         inSystem.ResetEffectFrameData(*m_Context);
     m_Flags.SetDirty(true);
 }
+
+CRegisteredString SEffect::GetError() const
+{
+    return m_error;
+}
+
+void SEffect::SetError(const CRegisteredString &error)
+{
+    m_error = error;
+}
diff --git a/src/runtimerender/graphobjects/Qt3DSRenderEffect.h b/src/runtimerender/graphobjects/Qt3DSRenderEffect.h
index 1ffc95f..e0db3d9 100644
--- a/src/runtimerender/graphobjects/Qt3DSRenderEffect.h
+++ b/src/runtimerender/graphobjects/Qt3DSRenderEffect.h
@@ -66,6 +66,8 @@ namespace render {
         typedef qt3ds::foundation::nvhash_map<CRegisteredString, SImage *> TImageMapHash;
         TImageMapHash *m_imageMaps;
 
+        CRegisteredString m_error;
+
         void Initialize();
 
         // If our active flag value changes, then we ask the effect manager
@@ -74,6 +76,9 @@ namespace render {
 
         void Reset(IEffectSystem &inSystem);
 
+        CRegisteredString GetError() const;
+        void SetError(const CRegisteredString &error);
+
         // Generic method used during serialization
         // to remap string and object pointers
         template <typename TRemapperType>
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
index 0eb7c51..cb9efea 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
@@ -43,6 +43,7 @@
 #include "Qt3DSRenderResourceManager.h"
 #include "Qt3DSTextRenderer.h"
 #include "Qt3DSRenderEffectSystem.h"
+#include "Qt3DSRenderContextCore.h"
 #include "render/Qt3DSRenderFrameBuffer.h"
 #include "render/Qt3DSRenderRenderBuffer.h"
 #include "Qt3DSOffscreenRenderKey.h"