Apply Gram-Schmidt when generating the world -> tangent matrix

Change-Id: I0801f5a4bb13efb77405c6af02367be0a7213d0f
Reviewed-by: Paul Lemire <paul.lemire@kdab.com>

1 files changed, 19 insertions, 11 deletions

diff --git a/src/extras/shaders/gl3/normaldiffusemap.vert b/src/extras/shaders/gl3/normaldiffusemap.vert
index 306a562fb..8da443e8d 100644
--- a/src/extras/shaders/gl3/normaldiffusemap.vert
+++ b/src/extras/shaders/gl3/normaldiffusemap.vert
@@ -17,23 +17,31 @@ uniform float texCoordScale;
 
 void main()
 {
-    // Pass through texture coordinates
+    // Pass through scaled texture coordinates
     texCoord = vertexTexCoord * texCoordScale;
 
     // Transform position, normal, and tangent to world coords
-    vec3 normal = normalize( modelNormalMatrix * vertexNormal );
-    vec3 tangent = normalize( modelNormalMatrix * vertexTangent.xyz );
-    worldPosition = vec3( modelMatrix * vec4( vertexPosition, 1.0 ) );
+    worldPosition = vec3(modelMatrix * vec4(vertexPosition, 1.0));
+    vec3 normal = normalize(modelNormalMatrix * vertexNormal);
+    vec3 tangent = normalize(vec3(modelMatrix * vec4(vertexTangent.xyz, 0.0)));
 
-    // Calculate binormal vector
-    vec3 binormal = normalize( cross( normal, tangent ) );
+    // Make the tangent truly orthogonal to the normal by using Gram-Schmidt.
+    // This allows to build the tangentMatrix below by simply transposing the
+    // tangent -> world space matrix (which would now be orthogonal)
+    tangent = normalize(tangent - dot(tangent, normal) * normal);
+
+    // Calculate binormal vector. No "real" need to renormalize it,
+    // as built by crossing two normal vectors.
+    // To orient the binormal correctly, use the fourth coordinate of the tangent,
+    // which is +1 for a right hand system, and -1 for a left hand system.
+    vec3 binormal = cross(normal, tangent) * vertexTangent.w;
 
     // Construct matrix to transform from eye coords to tangent space
-    tangentMatrix = mat3 (
-                tangent.x, binormal.x, normal.x,
-                tangent.y, binormal.y, normal.y,
-                tangent.z, binormal.z, normal.z );
+    tangentMatrix = mat3(
+        tangent.x, binormal.x, normal.x,
+        tangent.y, binormal.y, normal.y,
+        tangent.z, binormal.z, normal.z);
 
     // Calculate vertex position in clip coordinates
-    gl_Position = mvp * vec4( vertexPosition, 1.0 );
+    gl_Position = mvp * vec4(vertexPosition, 1.0);
 }