Indicate shader errors in Editor

Show shader errors with highlighting and tooltip in Project panel. Also
indicate error in Inspector panel with highlighting.

Change-Id: Ic601474d40058723872339f746165868307c596b
Task-id: QT3DS-3598
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>

14 files changed, 162 insertions, 27 deletions

diff --git a/src/Authoring/Qt3DStudio/Application/StudioApp.cpp b/src/Authoring/Qt3DStudio/Application/StudioApp.cpp
index e07e1b05..3a3db068 100644
--- a/src/Authoring/Qt3DStudio/Application/StudioApp.cpp
+++ b/src/Authoring/Qt3DStudio/Application/StudioApp.cpp
@@ -2059,6 +2059,13 @@ void CStudioApp::showPresentationIdEmptyWarning()
                                  Qt3DSMessageBox::ICON_WARNING, false);
 }
 
+void CStudioApp::showShaderCompileError(const QString &error)
+{
+    m_dialogs->DisplayMessageBox(tr("Warning"),
+                                 tr("Shader compile error.\n") + error,
+                                 Qt3DSMessageBox::ICON_WARNING, false);
+}
+
 void CStudioApp::showInvalidFilenameWarning()
 {
     m_dialogs->DisplayMessageBox(tr("Invalid filename"),
diff --git a/src/Authoring/Qt3DStudio/Application/StudioApp.h b/src/Authoring/Qt3DStudio/Application/StudioApp.h
index 8fccafcf..c3b90284 100644
--- a/src/Authoring/Qt3DStudio/Application/StudioApp.h
+++ b/src/Authoring/Qt3DStudio/Application/StudioApp.h
@@ -219,6 +219,7 @@ public:
     void showPresentationIdUniqueWarning();
     void showPresentationIdEmptyWarning();
     void showInvalidFilenameWarning();
+    void showShaderCompileError(const QString &error);
     void checkDeletedDatainputs(bool askFromUser);
     void saveDataInputsToProjectFile();
     void verifyDatainputBindings();
diff --git a/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.cpp b/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.cpp
index f0ca73ee..8912097b 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.cpp
+++ b/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.cpp
@@ -181,6 +181,7 @@ void InspectorControlModel::notifyPropertyChanged(qt3dsdm::Qt3DSDMInstanceHandle
             if (property->m_property == inProperty || imageInstance == inInstance) {
                 updatePropertyValue(property);
                 changed = true;
+                updateValidState(property);
             }
         }
     }
@@ -403,6 +404,7 @@ void InspectorControlModel::updateMaterialValues(const QStringList &values, int
                 Q_EMIT item->valuesChanged();
                 // Changing values resets the selected index, so pretend the value has also changed
                 Q_EMIT item->valueChanged();
+                updateValidState(item);
             }
         }
     }
@@ -715,7 +717,7 @@ InspectorControlBase *InspectorControlModel::createShaderItem(
             break;
         }
     }
-
+    updateValidState(item);
     return item;
 }
 
@@ -824,11 +826,6 @@ InspectorControlBase* InspectorControlModel::createItem(Qt3DSDMInspectable *insp
             std::bind(&InspectorControlModel::updateControlledToggleState, this, item)));
     }
 
-    if (item->m_propertyType == qt3dsdm::AdditionalMetaDataType::Import) {
-        item->m_connections.push_back(signalProvider->ConnectControlledToggled(
-            std::bind(&InspectorControlModel::updateValidState, this, item)));
-    }
-
     // synchronize the value itself
     updatePropertyValue(item);
     updateValidState(item);
@@ -912,7 +909,7 @@ void InspectorControlModel::updateAnimateToggleState(InspectorControlBase* inIte
     }
 }
 
-void InspectorControlModel::updateValidState(InspectorControlBase *inItem)
+void InspectorControlModel::updateValidState(InspectorControlBase *inItem) const
 {
     const auto bridge = g_StudioApp.GetCore()->GetDoc()->GetStudioSystem()
             ->GetClientDataModelBridge();
@@ -924,6 +921,22 @@ void InspectorControlModel::updateValidState(InspectorControlBase *inItem)
             Q_EMIT inItem->validDataChanged();
         }
     }
+
+    // Check the validity of shader.
+    if (inItem->m_title == tr("Shader")) {
+        auto err = g_StudioApp.getRenderer().getObjectError(inItem->m_instance);
+        if (!err.isEmpty()) {
+            inItem->m_tooltip = err;
+            inItem->m_valid = false;
+            Q_EMIT inItem->validDataChanged();
+            Q_EMIT inItem->tooltipChanged();
+        } else {
+            inItem->m_tooltip = tr("Shader being used");
+            inItem->m_valid = true;
+            Q_EMIT inItem->validDataChanged();
+            Q_EMIT inItem->tooltipChanged();
+        }
+    }
 }
 
 bool InspectorControlModel::isTreeRebuildRequired(CInspectableBase* inspectBase)
@@ -1445,6 +1458,8 @@ void InspectorControlModel::updatePropertyValue(InspectorControlBase *element) c
     // not the controlled flag nor the tooltip
     if (element->m_controllable)
           updateControlledToggleState(element);
+
+    updateValidState(element);
 }
 
 void InspectorControlModel::refreshRenderables()
@@ -1495,7 +1510,6 @@ void InspectorControlModel::refresh()
             if (property->m_property.Valid()) {
                 updatePropertyValue(property);
                 updateControlledToggleState(property);
-                updateValidState(property);
             }
         }
     }
@@ -1641,6 +1655,10 @@ void InspectorControlModel::setShaderValue(long instance, int handle, const QVar
     }
 
     saveIfMaterial(instance);
+
+    // Make sure that inspector panel is up-to-date with regards to result of shader compilation
+    // errors.
+    rebuildTree();
 }
 
 void InspectorControlModel::setMatDataValue(long instance, int handle, const QVariant &value)
diff --git a/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.h b/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.h
index 79e6613a..43b08207 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.h
+++ b/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlModel.h
@@ -143,7 +143,7 @@ public:
                                        bool controlled);
     void notifyPropertyChanged(qt3dsdm::Qt3DSDMInstanceHandle inInstance,
                                qt3dsdm::Qt3DSDMPropertyHandle inProperty);
-    void updateValidState(InspectorControlBase *inItem);
+    void updateValidState(InspectorControlBase *inItem) const;
 
     Q_INVOKABLE void setMaterialTypeValue(long instance, int handle, const QVariant &value);
     Q_INVOKABLE void setShaderValue(long instance, int handle, const QVariant &value);
diff --git a/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlView.qml b/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlView.qml
index d6e89ba3..99ed21cf 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlView.qml
+++ b/src/Authoring/Qt3DStudio/Palettes/Inspector/InspectorControlView.qml
@@ -394,11 +394,13 @@ Rectangle {
 
                                             readonly property var modelData: model.modelData
                                             text: model.modelData.title
-                                            // Color picked from DataInput icon
-                                            color: model.modelData.controlled?
-                                                       _dataInputColor
-                                                     : _parentView.titleColor(modelData.instance,
-                                                                              modelData.handle)
+                                            // Label is showing "controlled", "invalid data" or
+                                            // ordinary text color.
+                                            color: model.modelData.controlled
+                                                   ? _dataInputColor : model.modelData.validData
+                                                      ? _parentView.titleColor(modelData.instance,
+                                                                               modelData.handle)
+                                                         : _invalidDataIndicatorColor
 
                                             Layout.alignment: Qt.AlignTop
 
diff --git a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.cpp b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.cpp
index 92ea1b16..2287f294 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.cpp
+++ b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.cpp
@@ -50,6 +50,7 @@
 #include "IDragable.h"
 #include "IObjectReferenceHelper.h"
 #include "IDirectoryWatchingSystem.h"
+#include "IStudioRenderer.h"
 
 ProjectFileSystemModel::ProjectFileSystemModel(QObject *parent) : QAbstractListModel(parent)
     , m_model(new QFileSystemModel(this))
@@ -80,6 +81,7 @@ QHash<int, QByteArray> ProjectFileSystemModel::roleNames() const
     modelRoleNames.insert(ExpandedRole, "_expanded");
     modelRoleNames.insert(FileIdRole, "_fileId");
     modelRoleNames.insert(ExtraIconRole, "_extraIcon");
+    modelRoleNames.insert(HasWarningRole, "_hasWarning");
     return modelRoleNames;
 }
 
@@ -137,10 +139,16 @@ QVariant ProjectFileSystemModel::data(const QModelIndex &index, int role) const
         if (getIconType(path) & (OBJTYPE_PRESENTATION | OBJTYPE_QML_STREAM)) {
             if (presentationId(path).isEmpty())
                 return QStringLiteral("warning.png");
+        } else if (!item.error.isEmpty()) {
+                return QStringLiteral("warning.png");
         }
         return {};
     }
 
+    case HasWarningRole: {
+        return !item.error.isEmpty();
+    }
+
     default:
         return m_model->data(item.index, role);
     }
@@ -215,7 +223,39 @@ void ProjectFileSystemModel::updateReferences()
 
     m_references.insert(projectPath);
 
-    updateRoles({IsReferencedRole, Qt::DecorationRole});
+    checkShaders();
+
+    updateRoles({IsReferencedRole, Qt::DecorationRole, HasWarningRole, ExtraIconRole});
+}
+
+void ProjectFileSystemModel::showItemError(const QModelIndex index)
+{
+    const auto &item = m_items.at(index.row());
+    // Currently we only show shader compilation errors.
+    g_StudioApp.showShaderCompileError(item.error);
+}
+
+void ProjectFileSystemModel::checkShaders(const int startRow, const int endRow)
+{
+    auto bridge = g_StudioApp.GetCore()->GetDoc()->GetStudioSystem()->GetClientDataModelBridge();
+    qt3dsdm::TInstanceHandleList theShaderInstances(bridge->GetShaderInstances());
+
+    for (const auto &instance : theShaderInstances) {
+        auto sourcepath = bridge->GetSourcePath(instance);
+        const QString relativePath = g_StudioApp.GetCore()->GetDoc()
+                ->GetResolvedPathToDoc(sourcepath);
+        auto currRow = rowForPath(relativePath);
+
+        if (currRow >= startRow && startRow == -1 && currRow <= endRow)
+            continue;
+
+        auto err = g_StudioApp.getRenderer().getObjectError(instance);
+
+        if (!err.isEmpty() && currRow != -1)
+            m_items[currRow].error = bridge->GetSourcePath(instance) + QStringLiteral("\n\n") + err;
+        else if (currRow != -1)
+            m_items[currRow].error = QString();
+    }
 }
 
 /**
@@ -376,7 +416,8 @@ void ProjectFileSystemModel::updateProjectReferences()
     }
 
     m_projectReferencesUpdateMap.clear();
-    updateRoles({IsProjectReferencedRole});
+    checkShaders();
+    updateRoles({IsProjectReferencedRole, HasWarningRole, ExtraIconRole});
 }
 
 void ProjectFileSystemModel::getQmlAssets(const QObject *qmlNode,
@@ -576,6 +617,9 @@ void ProjectFileSystemModel::showModelChildItems(const QModelIndex &parentIndex,
 
     endInsertRows();
 
+    checkShaders(startRow, startRow + insertCount - 1);
+    updateRoles({HasWarningRole, ExtraIconRole});
+
     // also fetch children so we're notified when files are added or removed in immediate subdirs
     for (const auto &childIndex : rowsToInsert) {
         if (m_model->hasChildren(childIndex) && m_model->canFetchMore(childIndex))
@@ -675,6 +719,9 @@ void ProjectFileSystemModel::importUrls(const QList<QUrl> &urls, int row, bool a
     // Batch update all imported presentation nodes
     g_StudioApp.GetCore()->getProjectFile().addPresentationNodes(presentationNodes);
 
+    checkShaders(row, row);
+    updateRoles({HasWarningRole, ExtraIconRole});
+
     // Add new data inputs that are missing from project's data inputs. Duplicates are ignored,
     // even if they are different type.
     QMapIterator<QString, CDataInputDialogItem *> diIt(importedDataInputs);
diff --git a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.h b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.h
index ea583fcb..56cd56b5 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.h
+++ b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectFileSystemModel.h
@@ -59,7 +59,8 @@ public:
         DepthRole,
         ExpandedRole,
         FileIdRole,
-        ExtraIconRole
+        ExtraIconRole,
+        HasWarningRole
     };
 
     void setRootPath(const QString &path);
@@ -90,6 +91,8 @@ public:
     void asyncUpdateReferences();
     void onFilesChanged(const Q3DStudio::TFileModificationList &inFileModificationList);
     void clearModelData();
+    void checkShaders(const int startRow = -1, const int endRow = -1);
+    void showItemError(const QModelIndex index);
 
 Q_SIGNALS:
     void modelChanged(QAbstractItemModel *model);
@@ -141,6 +144,7 @@ private:
         bool expanded;
         TreeItem *parent;
         int childCount;
+        QString error;
     };
 
     QFileSystemModel *m_model = nullptr;
diff --git a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.cpp b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.cpp
index c2c84aa5..203952bc 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.cpp
+++ b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.cpp
@@ -318,6 +318,11 @@ void ProjectView::editPresentationId(int row, bool qmlStream)
     dlg.exec();
 }
 
+void ProjectView::showSpecificShaderError(const int row)
+{
+    m_ProjectModel->showItemError(m_ProjectModel->index(row));
+}
+
 void ProjectView::renamePresentation(int row, bool qmlStream)
 {
     QString relativePresPath = QDir(g_StudioApp.GetCore()->getProjectFile().getProjectPath())
diff --git a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.h b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.h
index b6a82e72..217aff1c 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.h
+++ b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.h
@@ -98,6 +98,7 @@ public:
     QString presentationId(int row) const;
     void setInitialPresentation(int row);
     Q_INVOKABLE void editPresentationId(int row, bool qmlStream);
+    Q_INVOKABLE void showSpecificShaderError(const int row);
     void renamePresentation(int row, bool qmlStream);
 
     // CPresentationChangeListener
diff --git a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.qml b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.qml
index 12d39048..d958152b 100644
--- a/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.qml
+++ b/src/Authoring/Qt3DStudio/Palettes/Project/ProjectView.qml
@@ -177,9 +177,11 @@ Rectangle {
                                 id: fileNameLabel
                                 text: _fileId ? fileName + " <" + _fileId + ">" : fileName;
                                 color: {
-                                    _isReferenced ? _textColor
-                                                  : _isProjectReferenced ? _projectReferencedColor
-                                                                         : _disabledColor
+                                    _hasWarning ? _invalidDataIndicatorColor :
+                                                  _isReferenced ? _textColor :
+                                                                  _isProjectReferenced ?
+                                                                      _projectReferencedColor
+                                                                    : _disabledColor
                                 }
                                 leftPadding: 2
 
@@ -211,13 +213,16 @@ Rectangle {
                                     hoverEnabled: true
                                     onPressed: delegateItem.handlePress(mouse, false)
                                     onClicked: delegateItem.handleClick(mouse)
-                                    onDoubleClicked: _parentView.editPresentationId(
-                                                         index, _parentView.isQmlStream(index))
+                                    onDoubleClicked:
+                                        _hasWarning ? _parentView.showSpecificShaderError(index)
+                                                    : _parentView.editPresentationId(
+                                                          index, _parentView.isQmlStream(index))
                                 }
                                 StyledTooltip {
-                                    text: _parentView.isPresentation(index)
-                                          ? qsTr("No presentation Id")
-                                          : qsTr("No Qml stream Id")
+                                    text: _hasWarning ? qsTr("Shader error")
+                                                      : _parentView.isPresentation(index)
+                                                        ? qsTr("No presentation Id")
+                                                        : qsTr("No Qml stream Id")
                                     enabled: warningMouseArea.containsMouse
                                 }
                             }
diff --git a/src/Authoring/Qt3DStudio/Render/IStudioRenderer.h b/src/Authoring/Qt3DStudio/Render/IStudioRenderer.h
index fc543a89..3faa65d9 100644
--- a/src/Authoring/Qt3DStudio/Render/IStudioRenderer.h
+++ b/src/Authoring/Qt3DStudio/Render/IStudioRenderer.h
@@ -67,6 +67,7 @@ public:
     virtual void SetGuidesEnabled(bool val) = 0;
     virtual bool AreGuidesEditable() const = 0;
     virtual void SetGuidesEditable(bool val) = 0;
+    virtual QString getObjectError(qt3dsdm::Qt3DSDMInstanceHandle theInstance) const = 0;
     // Setting the camera to -1 disables the edit cameras
     // So setting the camera to 0- (numcameras - 1) will set change the active
     // edit camera.
diff --git a/src/Authoring/Qt3DStudio/Render/StudioRenderer.cpp b/src/Authoring/Qt3DStudio/Render/StudioRenderer.cpp
index 320bca23..dd2c2519 100644
--- a/src/Authoring/Qt3DStudio/Render/StudioRenderer.cpp
+++ b/src/Authoring/Qt3DStudio/Render/StudioRenderer.cpp
@@ -417,6 +417,15 @@ struct SRendererImpl : public IStudioRenderer,
         Render();
     }
 
+    QString getObjectError(qt3dsdm::Qt3DSDMInstanceHandle theInstance) const
+    {
+        auto translator = m_Translation->GetOrCreateTranslator(theInstance);
+        if (translator)
+            return static_cast<SGraphObjectTranslator *>(translator)->GetError();
+        else
+            return {};
+    }
+
     void getPreviewFbo(QSize &outFboDim, qt3ds::QT3DSU32 &outFboTexture) override
     {
         if (m_Translation) {
diff --git a/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.cpp b/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.cpp
index c710dcd9..73619e5f 100644
--- a/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.cpp
+++ b/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.cpp
@@ -1161,10 +1161,13 @@ struct SEffectTranslator : public SDynamicObjectTranslator
     void SetActive(bool inActive) override
     {
         SEffect &theItem = static_cast<SEffect &>(GetGraphObject());
-        if (m_EffectSystem)
+        if (m_EffectSystem) {
             theItem.SetActive(inActive, *m_EffectSystem);
-        else
+            if (inActive && inActive != theItem.m_Flags.IsActive())
+                m_EffectSystem->SetEffectRequiresCompilation(theItem.m_ClassName, true);
+        } else {
             theItem.m_Flags.SetActive(inActive);
+        }
     }
 
     void ResetEffect() override
@@ -1173,6 +1176,19 @@ struct SEffectTranslator : public SDynamicObjectTranslator
         if (m_EffectSystem)
             theItem.Reset(*m_EffectSystem);
     }
+
+    const QString GetError() override
+    {
+        SEffect &theItem = static_cast<SEffect &>(GetGraphObject());
+        return QString::fromUtf8(theItem.GetError().c_str());
+    }
+
+    void SetError(const QString &error) override
+    {
+        SEffect &theItem = static_cast<SEffect &>(GetGraphObject());
+        theItem.SetError(m_EffectSystem->GetResourceManager().GetRenderContext()
+                         .GetStringTable().RegisterStr(error));
+    }
 };
 struct SCustomMaterialTranslator : public SDynamicObjectTranslator
 {
@@ -1206,10 +1222,27 @@ struct SCustomMaterialTranslator : public SDynamicObjectTranslator
             SCustomMaterial &theItem = static_cast<SCustomMaterial &>(GetGraphObject());
             if (inActive != theItem.m_Flags.IsActive()) {
                 theItem.m_Flags.SetActive(inActive);
+                // Force compilation in Studio if custom shader became active, so we know
+                // if it has any compilation errors.
+                if (inActive)
+                    m_MaterialSystem->setRequiresCompilation(theItem.m_ClassName, true);
+
                 m_MaterialSystem->OnMaterialActivationChange(theItem, inActive);
             }
         }
     }
+
+    const QString GetError() override
+    {
+        SCustomMaterial &theItem = static_cast<SCustomMaterial &>(GetGraphObject());
+        return QString::fromUtf8(theItem.GetError().c_str());
+    }
+
+    void SetError(const QString &error) override
+    {
+        SCustomMaterial &theItem = static_cast<SCustomMaterial &>(GetGraphObject());
+        theItem.SetError(m_MaterialSystem->getContext()->GetStringTable().RegisterStr(error));
+    }
 };
 struct SReferencedMaterialTranslator : public SGraphObjectTranslator
 {
diff --git a/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.h b/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.h
index 3dd9134f..f615664a 100644
--- a/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.h
+++ b/src/Authoring/Qt3DStudio/Render/StudioRendererTranslation.h
@@ -96,6 +96,8 @@ namespace studio {
         virtual void ClearChildren() = 0;
         virtual void AppendChild(SGraphObject &inChild) = 0;
         virtual void ResetEffect() {}
+        virtual const QString GetError() { return {}; }
+        virtual void SetError(const QString &error) { Q_UNUSED(error); }
         virtual SGraphObject &GetGraphObject() { return *m_GraphObject; }
         virtual SGraphObject &GetNonAliasedGraphObject() { return *m_GraphObject; }
         virtual qt3dsdm::Qt3DSDMInstanceHandle GetInstanceHandle() { return m_InstanceHandle; }