Add support for vertex shaders in the editor

The editor now supports custom material vertex shaders. For custom
materials with vertex shaders the uniforms the runtime provides are
changed to match the uniforms provided by runtime 2.x.
model_view_projection becomes modelViewProjection etc.

Task-number: QT3DS-3150
Change-Id: Ice58d4e0c197e628fe716d3608cdd58f6003d69c
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
Reviewed-by: Antti Määttä <antti.maatta@qt.io>

5 files changed, 214 insertions, 57 deletions

diff --git a/src/Authoring/QT3DSDM/Systems/Qt3DSDMMetaData.cpp b/src/Authoring/QT3DSDM/Systems/Qt3DSDMMetaData.cpp
index 8c599cfb..542b05d3 100644
--- a/src/Authoring/QT3DSDM/Systems/Qt3DSDMMetaData.cpp
+++ b/src/Authoring/QT3DSDM/Systems/Qt3DSDMMetaData.cpp
@@ -3094,6 +3094,10 @@ public:
                                      std::vector<SMetaDataLoadWarning> &outWarnings,
                                      eastl::string &shaderPrefix)
     {
+        eastl::string vertexUniforms;
+        eastl::string fragmentUniforms;
+        vertexUniforms += "#ifdef VERTEX_SHADER\n";
+        fragmentUniforms += "#ifdef FRAGMENT_SHADER\n";
         using namespace qt3ds::render::dynamic;
         ioObject.m_Properties.clear();
         ioObject.ClearEnumValueNames();
@@ -3116,27 +3120,32 @@ public:
                         m_StringTable.GetRenderStringTable().RegisterStr(theInfo.m_Name.c_str());
                 const char8_t *xmlName;
                 inStream.Att("name", xmlName);
+                const char8_t *stage;
+                inStream.Att("stage", stage);
+                eastl::string *uniforms = &fragmentUniforms;
+                if (AreEqual(stage, "vertex"))
+                    uniforms = &vertexUniforms;
                 if (AreEqual(xmlName, theNewDefinition.m_Name.c_str())) {
                     switch (theInfo.GetDataType()) {
                     case DataModelDataType::Bool:
                         theNewDefinition.m_DataType =
                                 qt3ds::render::NVRenderShaderDataTypes::QT3DSRenderBool;
-                        AppendShaderUniform("bool", theNewDefinition.m_Name.c_str(), shaderPrefix);
+                        AppendShaderUniform("bool", theNewDefinition.m_Name.c_str(), *uniforms);
                         break;
                     case DataModelDataType::Long:
                         theNewDefinition.m_DataType
                                 = qt3ds::render::NVRenderShaderDataTypes::QT3DSI32;
-                        AppendShaderUniform("int", theNewDefinition.m_Name.c_str(), shaderPrefix);
+                        AppendShaderUniform("int", theNewDefinition.m_Name.c_str(), *uniforms);
                         break;
                     case DataModelDataType::Float2:
                         theNewDefinition.m_DataType
                                 = qt3ds::render::NVRenderShaderDataTypes::QT3DSVec2;
-                        AppendShaderUniform("vec2", theNewDefinition.m_Name.c_str(), shaderPrefix);
+                        AppendShaderUniform("vec2", theNewDefinition.m_Name.c_str(), *uniforms);
                         break;
                     case DataModelDataType::Float3:
                         theNewDefinition.m_DataType
                                 = qt3ds::render::NVRenderShaderDataTypes::QT3DSVec3;
-                        AppendShaderUniform("vec3", theNewDefinition.m_Name.c_str(), shaderPrefix);
+                        AppendShaderUniform("vec3", theNewDefinition.m_Name.c_str(), *uniforms);
                         break;
                     case DataModelDataType::String:
                         if (theInfo.m_CompleteType == CompleteMetaDataType::Texture) {
@@ -3163,16 +3172,16 @@ public:
                             // Output macro so we can change the set of variables used for this
                             // independent of the
                             // meta data system.
-                            shaderPrefix.append("SNAPPER_SAMPLER2D(");
-                            shaderPrefix.append(theNewDefinition.m_Name.c_str());
-                            shaderPrefix.append(", ");
-                            shaderPrefix.append(theNewDefinition.m_Name.c_str());
-                            shaderPrefix.append(", ");
-                            shaderPrefix.append(filter);
-                            shaderPrefix.append(", ");
-                            shaderPrefix.append(clamp);
-                            shaderPrefix.append(", ");
-                            shaderPrefix.append("false )\n");
+                            uniforms->append("SNAPPER_SAMPLER2D(");
+                            uniforms->append(theNewDefinition.m_Name.c_str());
+                            uniforms->append(", ");
+                            uniforms->append(theNewDefinition.m_Name.c_str());
+                            uniforms->append(", ");
+                            uniforms->append(filter);
+                            uniforms->append(", ");
+                            uniforms->append(clamp);
+                            uniforms->append(", ");
+                            uniforms->append("false )\n");
                         } else if (theInfo.m_CompleteType == CompleteMetaDataType::StringList) {
                             theNewDefinition.m_DataType =
                                     qt3ds::render::NVRenderShaderDataTypes::QT3DSI32;
@@ -3188,54 +3197,54 @@ public:
                             theNewDefinition.m_EnumValueNames = VecToCRef(theBack);
                             theNewDefinition.m_IsEnumProperty = true;
                             AppendShaderUniform("int", theNewDefinition.m_Name.c_str(),
-                                                shaderPrefix);
+                                                *uniforms);
                         } else if (theInfo.m_CompleteType == CompleteMetaDataType::Image2D) {
                             theNewDefinition.m_DataType =
                                     qt3ds::render::NVRenderShaderDataTypes::NVRenderImage2DPtr;
                             const char8_t *format = "", *binding = "", *access = "readonly";
-                            shaderPrefix.append("layout(");
+                            uniforms->append("layout(");
                             inStream.Att("format", format);
-                            shaderPrefix.append(format);
+                            uniforms->append(format);
                             if (inStream.Att("binding", binding)) {
-                                shaderPrefix.append(", binding = ");
-                                shaderPrefix.append(binding);
+                                uniforms->append(", binding = ");
+                                uniforms->append(binding);
                             }
-                            shaderPrefix.append(") ");
+                            uniforms->append(") ");
 
                             // if we have format layout we cannot set an additional access qualifier
                             if (inStream.Att("access", access) && !AreEqual(format, ""))
-                                shaderPrefix.append(access);
+                                uniforms->append(access);
 
-                            shaderPrefix.append(" uniform image2D ");
-                            shaderPrefix.append(theNewDefinition.m_Name.c_str());
-                            shaderPrefix.append(";\n");
+                            uniforms->append(" uniform image2D ");
+                            uniforms->append(theNewDefinition.m_Name.c_str());
+                            uniforms->append(";\n");
                         } else if (theInfo.m_CompleteType == CompleteMetaDataType::Buffer) {
                             theNewDefinition.m_DataType =
                                     qt3ds::render::NVRenderShaderDataTypes::NVRenderDataBufferPtr;
                             const char8_t *align = "std140", *usage = "storage", *binding = "",
                                     *format = "float";
-                            shaderPrefix.append("layout(");
+                            uniforms->append("layout(");
 
                             inStream.Att("format", format);
                             inStream.Att("usage", usage);
                             if (AreEqual(usage, "storage")) {
                                 inStream.Att("align", align);
-                                shaderPrefix.append(align);
+                                uniforms->append(align);
 
                                 if (inStream.Att("binding", binding)) {
-                                    shaderPrefix.append(", binding = ");
-                                    shaderPrefix.append(binding);
+                                    uniforms->append(", binding = ");
+                                    uniforms->append(binding);
                                 }
 
-                                shaderPrefix.append(") ");
+                                uniforms->append(") ");
 
-                                shaderPrefix.append("buffer ");
-                                shaderPrefix.append(theNewDefinition.m_Name.c_str());
-                                shaderPrefix.append("\n{ \n");
-                                shaderPrefix.append(format);
-                                shaderPrefix.append(" ");
-                                shaderPrefix.append(theNewDefinition.m_Name.c_str());
-                                shaderPrefix.append("_data[]; \n};\n");
+                                uniforms->append("buffer ");
+                                uniforms->append(theNewDefinition.m_Name.c_str());
+                                uniforms->append("\n{ \n");
+                                uniforms->append(format);
+                                uniforms->append(" ");
+                                uniforms->append(theNewDefinition.m_Name.c_str());
+                                uniforms->append("_data[]; \n};\n");
                             } else {
                                 // currently we only handle storage counters
                                 QT3DS_ASSERT(false);
@@ -3247,7 +3256,7 @@ public:
                         // Fallthrough intentional
                     case DataModelDataType::Float:
                         theNewDefinition.m_DataType = qt3ds::render::NVRenderShaderDataTypes::QT3DSF32;
-                        AppendShaderUniform("float", theNewDefinition.m_Name.c_str(), shaderPrefix);
+                        AppendShaderUniform("float", theNewDefinition.m_Name.c_str(), *uniforms);
                         break;
                     }
                 } else {
@@ -3258,6 +3267,10 @@ public:
                 }
             }
         }
+        vertexUniforms += "#endif\n";
+        fragmentUniforms += "#endif\n";
+        shaderPrefix.append(vertexUniforms);
+        shaderPrefix.append(fragmentUniforms);
     }
 
     void LoadDynamicObjectShaders(IDOMReader &inStream, SMetaDataDynamicObjectImpl &ioObject,
diff --git a/src/Runtime/Source/Qt3DSRuntimeRender/Source/Qt3DSRenderCustomMaterialShaderGenerator.cpp b/src/Runtime/Source/Qt3DSRuntimeRender/Source/Qt3DSRenderCustomMaterialShaderGenerator.cpp
index ea15b91d..fe6a6dc3 100644
--- a/src/Runtime/Source/Qt3DSRuntimeRender/Source/Qt3DSRenderCustomMaterialShaderGenerator.cpp
+++ b/src/Runtime/Source/Qt3DSRuntimeRender/Source/Qt3DSRenderCustomMaterialShaderGenerator.cpp
@@ -487,8 +487,39 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
         return pCB;
     }
 
-    void GenerateVertexShader()
+    bool GenerateVertexShader(SShaderDefaultMaterialKey &, const char8_t *inShaderPathName)
     {
+        qt3ds::render::IDynamicObjectSystem &theDynamicSystem(
+            m_RenderContext.GetDynamicObjectSystem());
+        CRenderString theShaderBuffer;
+        const char8_t *vertSource = theDynamicSystem.GetShaderSource(
+            m_RenderContext.GetStringTable().RegisterStr(inShaderPathName), theShaderBuffer);
+
+        QT3DS_ASSERT(vertSource);
+        eastl::string srcString(vertSource);
+
+        // Check if the vertex shader portion already contains a main function
+        // The same string contains both the vertex and the fragment shader
+        // The last "#ifdef FRAGMENT_SHADER" should mark the start of the fragment shader
+        eastl_size_t fragmentDefStart = srcString.find("#ifdef FRAGMENT_SHADER");
+        eastl_size_t nextIndex = fragmentDefStart;
+        while (nextIndex != eastl::string::npos) {
+            nextIndex = srcString.find("#ifdef FRAGMENT_SHADER", nextIndex + 1);
+            if (nextIndex != eastl::string::npos)
+                fragmentDefStart = nextIndex;
+        }
+        eastl_size_t mainStart = srcString.find("void main()");
+
+        if (mainStart != eastl::string::npos && (fragmentDefStart == eastl::string::npos
+                                                 || mainStart < fragmentDefStart)) {
+            TShaderGeneratorStageFlags stages(IShaderProgramGenerator::DefaultFlags());
+            ProgramGenerator().BeginProgram(stages);
+            IDefaultMaterialVertexPipeline &vertexShader(VertexGenerator());
+            vertexShader << "#define VERTEX_SHADER\n\n";
+            vertexShader << srcString.data() << Endl;
+            return true;
+        }
+
         // vertex displacement
         QT3DSU32 imageIdx = 0;
         SRenderableImage *displacementImage = NULL;
@@ -505,6 +536,7 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
 
         // the pipeline opens/closes up the shaders stages
         VertexGenerator().BeginVertexGeneration(displacementImageIdx, displacementImage);
+        return false;
     }
 
     SShaderGeneratorGeneratedShader &GetShaderForProgram(NVRenderShaderProgram &inProgram)
@@ -989,7 +1021,8 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
         inFragmentShader << "}\n\n";
     }
 
-    void GenerateFragmentShader(SShaderDefaultMaterialKey &, const char8_t *inShaderPathName)
+    void GenerateFragmentShader(SShaderDefaultMaterialKey &, const char8_t *inShaderPathName,
+                                bool hasCustomVertexShader)
     {
         qt3ds::render::IDynamicObjectSystem &theDynamicSystem(
             m_RenderContext.GetDynamicObjectSystem());
@@ -1017,10 +1050,12 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
             }
         }
 
-        VertexGenerator().GenerateUVCoords(0);
-        // for lightmaps we expect a second set of uv coordinates
-        if (hasLightmaps)
-            VertexGenerator().GenerateUVCoords(1);
+        if (!hasCustomVertexShader) {
+            VertexGenerator().GenerateUVCoords(0);
+            // for lightmaps we expect a second set of uv coordinates
+            if (hasLightmaps)
+                VertexGenerator().GenerateUVCoords(1);
+        }
 
         IDefaultMaterialVertexPipeline &vertexShader(VertexGenerator());
         IShaderStageGenerator &fragmentShader(FragmentGenerator());
@@ -1038,7 +1073,26 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
 
         fragmentShader << "#define FRAGMENT_SHADER\n\n";
 
-        if (srcString.find("void main()") == eastl::string::npos)
+        // Check if the fragment shader portion already contains a main function
+        // The same string contains both the vertex and the fragment shader
+        // The last "#ifdef FRAGMENT_SHADER" should mark the start of the fragment shader
+        eastl_size_t fragmentDefStart = srcString.find("#ifdef FRAGMENT_SHADER");
+        eastl_size_t nextIndex = fragmentDefStart;
+        while (nextIndex != eastl::string::npos) {
+            nextIndex = srcString.find("#ifdef FRAGMENT_SHADER", nextIndex + 1);
+            if (nextIndex != eastl::string::npos)
+                fragmentDefStart = nextIndex;
+        }
+        eastl_size_t mainStart = srcString.find("void main()");
+        if (fragmentDefStart == eastl::string::npos)
+            return;
+
+        if (mainStart != eastl::string::npos && mainStart < fragmentDefStart)
+            mainStart = srcString.find("void main()", mainStart + 1);
+
+        bool hasCustomFragmentShader = mainStart != eastl::string::npos;
+
+        if (!hasCustomFragmentShader)
             fragmentShader.AddInclude("evalLightmaps.glsllib");
 
         // check dielectric materials
@@ -1051,16 +1105,15 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
 
         fragmentShader << srcString.data() << Endl;
 
-        if (srcString.find("void main()") != eastl::string::npos) // If a "main()" is already
-                                                                  // written, we'll assume that the
-                                                                  // shader
-        { // pass is already written out and we don't need to add anything.
-            // Nothing beyond the basics, anyway
-            vertexShader.GenerateWorldNormal();
-            vertexShader.GenerateVarTangentAndBinormal();
-            vertexShader.GenerateWorldPosition();
+        if (hasCustomFragmentShader) {
+            fragmentShader << "#define FRAGMENT_SHADER\n\n";
+            if (!hasCustomVertexShader) {
+                vertexShader.GenerateWorldNormal();
+                vertexShader.GenerateVarTangentAndBinormal();
+                vertexShader.GenerateWorldPosition();
 
-            vertexShader.GenerateViewVector();
+                vertexShader.GenerateViewVector();
+            }
             return;
         }
 
@@ -1153,14 +1206,31 @@ struct SShaderGenerator : public ICustomMaterialShaderGenerator
         SShaderDefaultMaterialKey theKey(Key());
         theKey.ToString(m_GeneratedShaderString, m_DefaultMaterialShaderKeyProperties);
 
-        GenerateVertexShader();
-        GenerateFragmentShader(theKey, inCustomMaterialName);
+        bool hasCustomVertexShader = GenerateVertexShader(theKey, inCustomMaterialName);
+        GenerateFragmentShader(theKey, inCustomMaterialName, hasCustomVertexShader);
 
         VertexGenerator().EndVertexGeneration();
         VertexGenerator().EndFragmentGeneration();
 
-        return ProgramGenerator().CompileGeneratedShader(m_GeneratedShaderString.c_str(),
-                                                         SShaderCacheProgramFlags(), FeatureSet());
+        NVRenderShaderProgram *program = ProgramGenerator().CompileGeneratedShader(
+                    m_GeneratedShaderString.c_str(), SShaderCacheProgramFlags(), FeatureSet());
+        if (program && hasCustomVertexShader) {
+            // Change uniforms names to match runtime 2.x uniforms
+            SShaderGeneratorGeneratedShader &shader(GetShaderForProgram(*program));
+            shader.m_ModelMatrix = NVRenderCachedShaderProperty<QT3DSMat44>("modelMatrix",
+                                                                            *program);
+            shader.m_ViewProjMatrix = NVRenderCachedShaderProperty<QT3DSMat44>(
+                        "modelViewProjection", *program);
+            shader.m_ViewMatrix = NVRenderCachedShaderProperty<QT3DSMat44>("viewMatrix", *program);
+            shader.m_NormalMatrix = NVRenderCachedShaderProperty<QT3DSMat33>("modelNormalMatrix",
+                                                                             *program);
+            shader.m_ProjMatrix = NVRenderCachedShaderProperty<QT3DSMat44>("viewProjectionMatrix",
+                                                                           *program);
+            shader.m_ViewportMatrix = NVRenderCachedShaderProperty<QT3DSMat44>("viewportMatrix",
+                                                                               *program);
+            shader.m_CameraPos = NVRenderCachedShaderProperty<QT3DSVec3>("eyePosition", *program);
+        }
+        return program;
     }
 
     virtual NVRenderShaderProgram *
diff --git a/tests/scenes/customvertex/customvertex.uia b/tests/scenes/customvertex/customvertex.uia
new file mode 100644
index 00000000..84488102
--- /dev/null
+++ b/tests/scenes/customvertex/customvertex.uia
@@ -0,0 +1,15 @@
+<?xml version='1.0' encoding='utf-8'?>
+<application xmlns="http://qt.io/qt3dstudio/uia">
+    <assets initial="customvertex">
+        <presentation id="customvertex" src="presentations/customvertex.uip"/>
+    </assets>
+    <statemachine ref="#logic">
+        <visual-states>
+            <state ref="Initial">
+                <enter>
+                    <goto-slide rel="next" element="main:Scene"/>
+                </enter>
+            </state>
+        </visual-states>
+    </statemachine>
+</application>
diff --git a/tests/scenes/customvertex/materials/simple.shader b/tests/scenes/customvertex/materials/simple.shader
new file mode 100644
index 00000000..5e267cfa
--- /dev/null
+++ b/tests/scenes/customvertex/materials/simple.shader
@@ -0,0 +1,26 @@
+<Material name="simple" version="1.0">
+    <MetaData>
+        <Property formalName="Scale" name="scale" type="Vector" default="1 1 1" stage="vertex" category="Material" />
+        <Property formalName="Color" name="color" type="Color" default="1 1 1" stage="fragment" category="Material" />
+    </MetaData>
+    <Shaders type="GLSL" version="330">
+        <Shader>
+            <VertexShader>
+                attribute vec3 attr_pos;
+                uniform mat4 modelViewProjection;
+
+                void main() {
+                    gl_Position = modelViewProjection * vec4(attr_pos * scale, 1.0);
+            </VertexShader>
+            <FragmentShader>
+                void main() {
+                    fragOutput = vec4(color, 1.0);
+            </FragmentShader>
+        </Shader>
+    </Shaders>
+<Passes>
+    <Pass>
+    </Pass>
+</Passes>
+</Material>
+
diff --git a/tests/scenes/customvertex/presentations/customvertex.uip b/tests/scenes/customvertex/presentations/customvertex.uip
new file mode 100644
index 00000000..eba06daa
--- /dev/null
+++ b/tests/scenes/customvertex/presentations/customvertex.uip
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<UIP version="5" >
+	<Project >
+		<ProjectSettings author="" company="" presentationWidth="1920" presentationHeight="1080" maintainAspect="False" preferKtx="False" >
+			<CustomColors count="16" >#7391ff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff #ffffff</CustomColors>
+		</ProjectSettings>
+		<Classes >
+			<CustomMaterial id="simple" name="simple" sourcepath="../materials/simple.shader" />
+		</Classes>
+		<Graph >
+			<Scene id="Scene" >
+				<Layer id="Layer" variants="" >
+					<Camera id="Camera" />
+					<Light id="Light" />
+					<Model id="Sphere" >
+						<CustomMaterial id="simple_001" class="#simple" />
+					</Model>
+				</Layer>
+			</Scene>
+		</Graph>
+		<Logic >
+			<State name="Master Slide" component="#Scene" >
+				<Add ref="#Layer" disabledepthprepass="True" />
+				<Add ref="#Camera" />
+				<Add ref="#Light" />
+				<State id="Scene-Slide1" name="Slide1" >
+					<Add ref="#Sphere" name="Sphere" position="0 0 0" sourcepath="#Sphere" />
+					<Add ref="#simple_001" name="Material" color="0.333333 0.666667 0" scale="1 2 1" />
+				</State>
+			</State>
+		</Logic>
+	</Project>
+</UIP>