Fix planets-qml inner planets shading

* The shadow light was using the undefined matrix property which
  resulted in a matrix full of NaN's in the shader. This caused
  everything to be fully in shadows for the inner planets. Fixed
  to use the correct viewMatrix property.

* After fixing the above, it became clear that the inner planets
  were being shadowed with lots of artifacts. The only shadow
  casters are the planets Saturn and Uranus and their rings. In
  this configuration it makes no sense to try to shadow the other
  planets as this leads to bad shadow acne due to the huge
  distances involved vs the limited precision of the shadow
  depth texture. Fixed these artifacts by only applying shadows
  to Saturn and Uranus.

Task-number: QTBUG-53751
Change-Id: I96ab08117c9988c40f083a1345ec7f7d1b0f7ce6
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>
Reviewed-by: Paul Lemire <paul.lemire@kdab.com>

11 files changed, 185 insertions, 59 deletions

diff --git a/examples/qt3d/planets-qml/PlanetsLight.qml b/examples/qt3d/planets-qml/PlanetsLight.qml
index ea73fe9db..86e4111c3 100644
--- a/examples/qt3d/planets-qml/PlanetsLight.qml
+++ b/examples/qt3d/planets-qml/PlanetsLight.qml
@@ -63,7 +63,7 @@ Entity {
 
     readonly property Camera lightCamera: lightCamera
     readonly property matrix4x4 lightViewProjection:
-        lightCamera.projectionMatrix.times(lightCamera.matrix)
+        lightCamera.projectionMatrix.times(lightCamera.viewMatrix)
 
     Camera {
         id: lightCamera
diff --git a/examples/qt3d/planets-qml/ShadowEffect.qml b/examples/qt3d/planets-qml/ShadowEffect.qml
index c09ebebc5..a5d414c47 100644
--- a/examples/qt3d/planets-qml/ShadowEffect.qml
+++ b/examples/qt3d/planets-qml/ShadowEffect.qml
@@ -90,8 +90,8 @@ Effect {
         filterKeys: [ forwardkey ]
 
         shaderProgram: ShaderProgram {
-            vertexShaderCode:   loadSource("qrc:/shaders/gl3/planetD.vert")
-            fragmentShaderCode: loadSource("qrc:/shaders/gl3/planetD.frag")
+            vertexShaderCode:   loadSource("qrc:/shaders/gl3/planetDShadow.vert")
+            fragmentShaderCode: loadSource("qrc:/shaders/gl3/planetDShadow.frag")
         }
 
         // no special render state set => use the default set of states
diff --git a/examples/qt3d/planets-qml/planets-qml.qrc b/examples/qt3d/planets-qml/planets-qml.qrc
index 4b6bd433d..ebac0bcd5 100644
--- a/examples/qt3d/planets-qml/planets-qml.qrc
+++ b/examples/qt3d/planets-qml/planets-qml.qrc
@@ -35,5 +35,7 @@
         <file>shaders/es2/planetDB.vert</file>
         <file>shaders/es2/planetDS.frag</file>
         <file>shaders/es2/planetDSB.frag</file>
+        <file>shaders/gl3/planetDShadow.frag</file>
+        <file>shaders/gl3/planetDShadow.vert</file>
     </qresource>
 </RCC>
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetD.frag b/examples/qt3d/planets-qml/shaders/gl3/planetD.frag
index 74da1f37a..667c6c9aa 100644
--- a/examples/qt3d/planets-qml/shaders/gl3/planetD.frag
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetD.frag
@@ -49,10 +49,6 @@ uniform float opacity;      // Alpha channel
 
 uniform sampler2D diffuseTexture;
 
-uniform sampler2DShadow shadowMapTexture;
-
-in vec4 positionInLightSpace;
-
 in vec3 position;
 in vec3 normal;
 in vec2 texCoord;
@@ -91,11 +87,8 @@ void main()
     vec2 flipYTexCoord = texCoord;
     flipYTexCoord.y = 1.0 - texCoord.y;
 
-    float shadowMapSample = textureProj(shadowMapTexture, positionInLightSpace);
-
     vec3 result = lightIntensity * ka * texture(diffuseTexture, flipYTexCoord).rgb;
-    if (shadowMapSample > 0)
-        result = dModel(flipYTexCoord);
+    result += dModel(flipYTexCoord);
 
     float alpha = opacity * texture(diffuseTexture, flipYTexCoord).a;
 
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetD.vert b/examples/qt3d/planets-qml/shaders/gl3/planetD.vert
index 41a1db6e4..6070eb547 100644
--- a/examples/qt3d/planets-qml/shaders/gl3/planetD.vert
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetD.vert
@@ -41,12 +41,10 @@ in vec3 vertexPosition;
 in vec3 vertexNormal;
 in vec2 vertexTexCoord;
 
-out vec4 positionInLightSpace;
 out vec3 position;
 out vec3 normal;
 out vec2 texCoord;
 
-uniform mat4 lightViewProjection;
 uniform mat4 modelMatrix;
 uniform mat4 modelView;
 uniform mat3 modelViewNormal;
@@ -56,13 +54,6 @@ uniform float texCoordScale;
 
 void main()
 {
-    const mat4 shadowMatrix = mat4(0.5, 0.0, 0.0, 0.0,
-                                   0.0, 0.5, 0.0, 0.0,
-                                   0.0, 0.0, 0.5, 0.0,
-                                   0.5, 0.5, 0.5, 1.0);
-
-    positionInLightSpace = shadowMatrix * lightViewProjection * modelMatrix * vec4(vertexPosition, 1.0);
-
     texCoord = vertexTexCoord * texCoordScale;
     normal = normalize(modelViewNormal * vertexNormal);
     position = vec3(modelView * vec4(vertexPosition, 1.0));
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetDB.frag b/examples/qt3d/planets-qml/shaders/gl3/planetDB.frag
index bf53a127b..ea1230d97 100644
--- a/examples/qt3d/planets-qml/shaders/gl3/planetDB.frag
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetDB.frag
@@ -50,10 +50,6 @@ uniform float opacity;      // Alpha channel
 uniform sampler2D diffuseTexture;
 uniform sampler2D normalTexture;
 
-uniform sampler2DShadow shadowMapTexture;
-
-in vec4 positionInLightSpace;
-
 in vec3 lightDir;
 in vec3 viewDir;
 in vec2 texCoord;
@@ -88,8 +84,6 @@ void main()
     vec2 flipYTexCoord = texCoord;
     flipYTexCoord.y = 1.0 - texCoord.y;
 
-    float shadowMapSample = textureProj(shadowMapTexture, positionInLightSpace);
-
     // Sample the textures at the interpolated texCoords
     vec4 normal = 2.0 * texture(normalTexture, flipYTexCoord) - vec4(1.0);
 
@@ -97,10 +91,8 @@ void main()
 
     // Calculate the lighting model, keeping the specular component separate
     vec3 ambientAndDiff, spec;
-    if (shadowMapSample > 0) {
-        dbModel(normalize(normal.xyz), flipYTexCoord, ambientAndDiff, spec);
-        result = ambientAndDiff + spec;
-    }
+    dbModel(normalize(normal.xyz), flipYTexCoord, ambientAndDiff, spec);
+    result = ambientAndDiff + spec;
 
     // Combine spec with ambient+diffuse for final fragment color
     fragColor = vec4(result, opacity);
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetDB.vert b/examples/qt3d/planets-qml/shaders/gl3/planetDB.vert
index e30394aa3..17f925dc6 100644
--- a/examples/qt3d/planets-qml/shaders/gl3/planetDB.vert
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetDB.vert
@@ -42,13 +42,11 @@ in vec3 vertexNormal;
 in vec2 vertexTexCoord;
 in vec4 vertexTangent;
 
-out vec4 positionInLightSpace;
 out vec3 lightDir;
 out vec3 viewDir;
 out vec2 texCoord;
 
 uniform mat4 viewMatrix;
-uniform mat4 lightViewProjection;
 uniform mat4 modelMatrix;
 uniform mat4 modelView;
 uniform mat3 modelViewNormal;
@@ -60,13 +58,6 @@ uniform vec3 lightPosition;
 
 void main()
 {
-    const mat4 shadowMatrix = mat4(0.5, 0.0, 0.0, 0.0,
-                                   0.0, 0.5, 0.0, 0.0,
-                                   0.0, 0.0, 0.5, 0.0,
-                                   0.5, 0.5, 0.5, 1.0);
-
-    positionInLightSpace = shadowMatrix * lightViewProjection * modelMatrix * vec4(vertexPosition, 1.0);
-
     // Pass through texture coordinates
     texCoord = vertexTexCoord * texCoordScale;
 
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetDS.frag b/examples/qt3d/planets-qml/shaders/gl3/planetDS.frag
index 03922362f..bba96bd4b 100644
--- a/examples/qt3d/planets-qml/shaders/gl3/planetDS.frag
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetDS.frag
@@ -49,10 +49,6 @@ uniform float opacity;      // Alpha channel
 uniform sampler2D diffuseTexture;
 uniform sampler2D specularTexture;
 
-uniform sampler2DShadow shadowMapTexture;
-
-in vec4 positionInLightSpace;
-
 in vec3 position;
 in vec3 normal;
 in vec2 texCoord;
@@ -92,11 +88,8 @@ void main()
     vec2 flipYTexCoord = texCoord;
     flipYTexCoord.y = 1.0 - texCoord.y;
 
-    float shadowMapSample = textureProj(shadowMapTexture, positionInLightSpace);
-
     vec3 result = lightIntensity * ka * texture(diffuseTexture, flipYTexCoord).rgb;
-    if (shadowMapSample > 0)
-        result = dsModel(flipYTexCoord);
+    result += dsModel(flipYTexCoord);
 
     fragColor = vec4(result, opacity * texture(diffuseTexture, flipYTexCoord).a);
 }
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetDSB.frag b/examples/qt3d/planets-qml/shaders/gl3/planetDSB.frag
index deb9565d4..1982ab17b 100644
--- a/examples/qt3d/planets-qml/shaders/gl3/planetDSB.frag
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetDSB.frag
@@ -50,10 +50,6 @@ uniform sampler2D diffuseTexture;
 uniform sampler2D specularTexture;
 uniform sampler2D normalTexture;
 
-uniform sampler2DShadow shadowMapTexture;
-
-in vec4 positionInLightSpace;
-
 in vec3 lightDir;
 in vec3 viewDir;
 in vec2 texCoord;
@@ -91,19 +87,13 @@ void main()
     vec2 flipYTexCoord = texCoord;
     flipYTexCoord.y = 1.0 - texCoord.y;
 
-    float shadowMapSample = textureProj(shadowMapTexture, positionInLightSpace);
-
     // Sample the textures at the interpolated texCoords
     vec4 normal = 2.0 * texture(normalTexture, flipYTexCoord) - vec4(1.0);
 
-    vec3 result = lightIntensity * ka * texture(diffuseTexture, flipYTexCoord).rgb;
-
     // Calculate the lighting model, keeping the specular component separate
     vec3 ambientAndDiff, spec;
-    if (shadowMapSample > 0) {
-        dsbModel(normalize(normal.xyz), flipYTexCoord, ambientAndDiff, spec);
-        result = ambientAndDiff + spec;
-    }
+    dsbModel(normalize(normal.xyz), flipYTexCoord, ambientAndDiff, spec);
+    vec3 result = ambientAndDiff + spec;
 
     // Combine spec with ambient+diffuse for final fragment color
     fragColor = vec4(result, opacity);
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetDShadow.frag b/examples/qt3d/planets-qml/shaders/gl3/planetDShadow.frag
new file mode 100644
index 000000000..8f722388f
--- /dev/null
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetDShadow.frag
@@ -0,0 +1,103 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Klaralvdalens Datakonsult AB (KDAB).
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#version 150 core
+
+uniform mat4 viewMatrix;
+
+uniform vec3 lightPosition;
+uniform vec3 lightIntensity;
+
+uniform vec3 ka;            // Ambient reflectivity
+uniform vec3 ks;            // Specular reflectivity
+uniform float shininess;    // Specular shininess factor
+uniform float opacity;      // Alpha channel
+
+uniform sampler2D diffuseTexture;
+
+uniform sampler2DShadow shadowMapTexture;
+
+in vec4 positionInLightSpace;
+
+in vec3 position;
+in vec3 normal;
+in vec2 texCoord;
+
+out vec4 fragColor;
+
+vec3 dModel(const in vec2 flipYTexCoord)
+{
+    // Calculate the vector from the light to the fragment
+    vec3 s = normalize(vec3(viewMatrix * vec4(lightPosition, 1.0)) - position);
+
+    // Calculate the vector from the fragment to the eye position
+    // (origin since this is in "eye" or "camera" space)
+    vec3 v = normalize(-position);
+
+    // Reflect the light beam using the normal at this fragment
+    vec3 r = reflect(-s, normal);
+
+    // Calculate the diffuse component
+    float diffuse = max(dot(s, normal), 0.0);
+
+    // Calculate the specular component
+    float specular = 0.0;
+    if (dot(s, normal) > 0.0)
+        specular = (shininess / (8.0 * 3.14)) * pow(max(dot(r, v), 0.0), shininess);
+
+    // Lookup diffuse and specular factors
+    vec3 diffuseColor = texture(diffuseTexture, flipYTexCoord).rgb;
+
+    // Combine the ambient, diffuse and specular contributions
+    return lightIntensity * ((ka + diffuse) * diffuseColor + specular * ks);
+}
+
+void main()
+{
+    vec2 flipYTexCoord = texCoord;
+    flipYTexCoord.y = 1.0 - texCoord.y;
+
+    float shadowMapSample = textureProj(shadowMapTexture, positionInLightSpace);
+
+    vec3 result = lightIntensity * ka * texture(diffuseTexture, flipYTexCoord).rgb;
+    if (shadowMapSample > 0)
+        result += dModel(flipYTexCoord);
+
+    float alpha = opacity * texture(diffuseTexture, flipYTexCoord).a;
+
+    fragColor = vec4(result, alpha);
+}
diff --git a/examples/qt3d/planets-qml/shaders/gl3/planetDShadow.vert b/examples/qt3d/planets-qml/shaders/gl3/planetDShadow.vert
new file mode 100644
index 000000000..41a1db6e4
--- /dev/null
+++ b/examples/qt3d/planets-qml/shaders/gl3/planetDShadow.vert
@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Klaralvdalens Datakonsult AB (KDAB).
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#version 150 core
+
+in vec3 vertexPosition;
+in vec3 vertexNormal;
+in vec2 vertexTexCoord;
+
+out vec4 positionInLightSpace;
+out vec3 position;
+out vec3 normal;
+out vec2 texCoord;
+
+uniform mat4 lightViewProjection;
+uniform mat4 modelMatrix;
+uniform mat4 modelView;
+uniform mat3 modelViewNormal;
+uniform mat4 mvp;
+
+uniform float texCoordScale;
+
+void main()
+{
+    const mat4 shadowMatrix = mat4(0.5, 0.0, 0.0, 0.0,
+                                   0.0, 0.5, 0.0, 0.0,
+                                   0.0, 0.0, 0.5, 0.0,
+                                   0.5, 0.5, 0.5, 1.0);
+
+    positionInLightSpace = shadowMatrix * lightViewProjection * modelMatrix * vec4(vertexPosition, 1.0);
+
+    texCoord = vertexTexCoord * texCoordScale;
+    normal = normalize(modelViewNormal * vertexNormal);
+    position = vec3(modelView * vec4(vertexPosition, 1.0));
+
+    gl_Position = mvp * vec4(vertexPosition, 1.0);
+}