Work around mis-optimisation on QNX/SGX in AA vertex shader

The new AA algorithm which is implemented in the vertex shader of
SmoothColorMaterialShader and SmoothTextureMaterialShader exposes
a driver bug in the glsl compiler/optimiser for the GPU used in
the BlackBerry PlayBook.

The bug results in the if (scale < 0.0) scale = 1.0 code always
being executed even when the condition is false. This leads to
massive corrupion of Image, Rectangle, and BorderImage elements
when the antialiasing property is enabled.

This commit works around the compiler bug by refactoring the
vertex shader to explicitly include an else clause. The check for
negative values is now performed on the numerator only as the
denominator is dot(dir, dir) which is always positive (square of
magnitude of dir vector).

This gives the correct behaviour on all platforms (that I have
access to).

Change-Id: I236542c9b59ff2915e95cbd9300b442be316bba9
Reviewed-by: Samuel Rødal <samuel.rodal@digia.com>

2 files changed, 24 insertions, 8 deletions

diff --git a/src/quick/scenegraph/qsgdefaultimagenode.cpp b/src/quick/scenegraph/qsgdefaultimagenode.cpp
index b165b2db12..d9c04e1959 100644
--- a/src/quick/scenegraph/qsgdefaultimagenode.cpp
+++ b/src/quick/scenegraph/qsgdefaultimagenode.cpp
@@ -159,8 +159,12 @@ const char *SmoothTextureMaterialShader::vertexShader() const
             "        highp vec2 dir = delta.xy * pos.w - pos.xy * delta.w; \n"
             "        highp vec2 ndir = .5 * pixelSize * normalize(dir / pixelSize);  \n"
             "        dir -= ndir * delta.w * pos.w; \n"
-            "        highp float scale = min(1., dot(dir, ndir * pos.w * pos.w) / dot(dir, dir)); \n"
-            "        if (scale < 0.) scale = 1.; \n"
+            "        highp float numerator = dot(dir, ndir * pos.w * pos.w); \n"
+            "        highp float scale = 0.0; \n"
+            "        if (numerator < 0.0) \n"
+            "            scale = 1.0; \n"
+            "        else \n"
+            "            scale = min(1.0, numerator / dot(dir, dir)); \n"
             "        gl_Position += scale * delta; \n"
             "        texCoord.x += scale * texCoordOffset.x; \n"
             "    } \n"
@@ -170,8 +174,12 @@ const char *SmoothTextureMaterialShader::vertexShader() const
             "        highp vec2 dir = delta.xy * pos.w - pos.xy * delta.w; \n"
             "        highp vec2 ndir = .5 * pixelSize * normalize(dir / pixelSize);  \n"
             "        dir -= ndir * delta.w * pos.w; \n"
-            "        highp float scale = min(1., dot(dir, ndir * pos.w * pos.w) / dot(dir, dir)); \n"
-            "        if (scale < 0.) scale = 1.; \n"
+            "        highp float numerator = dot(dir, ndir * pos.w * pos.w); \n"
+            "        highp float scale = 0.0; \n"
+            "        if (numerator < 0.0) \n"
+            "            scale = 1.0; \n"
+            "        else \n"
+            "            scale = min(1.0, numerator / dot(dir, dir)); \n"
             "        gl_Position += scale * delta; \n"
             "        texCoord.y += scale * texCoordOffset.y; \n"
             "    } \n"
diff --git a/src/quick/scenegraph/qsgdefaultrectanglenode.cpp b/src/quick/scenegraph/qsgdefaultrectanglenode.cpp
index 2da8e3cd83..dab855064e 100644
--- a/src/quick/scenegraph/qsgdefaultrectanglenode.cpp
+++ b/src/quick/scenegraph/qsgdefaultrectanglenode.cpp
@@ -175,8 +175,12 @@ const char *SmoothColorMaterialShader::vertexShader() const
             "        highp vec2 dir = delta.xy * pos.w - pos.xy * delta.w; \n"
             "        highp vec2 ndir = .5 * pixelSize * normalize(dir / pixelSize);  \n"
             "        dir -= ndir * delta.w * pos.w; \n"
-            "        highp float scale = min(1., dot(dir, ndir * pos.w * pos.w) / dot(dir, dir)); \n"
-            "        if (scale < 0.) scale = 1.; \n"
+            "        highp float numerator = dot(dir, ndir * pos.w * pos.w); \n"
+            "        highp float scale = 0.0; \n"
+            "        if (numerator < 0.0) \n"
+            "            scale = 1.0; \n"
+            "        else \n"
+            "            scale = min(1.0, numerator / dot(dir, dir)); \n"
             "        gl_Position += scale * delta; \n"
             "    } \n"
 
@@ -185,8 +189,12 @@ const char *SmoothColorMaterialShader::vertexShader() const
             "        highp vec2 dir = delta.xy * pos.w - pos.xy * delta.w; \n"
             "        highp vec2 ndir = .5 * pixelSize * normalize(dir / pixelSize);  \n"
             "        dir -= ndir * delta.w * pos.w; \n"
-            "        highp float scale = min(1., dot(dir, ndir * pos.w * pos.w) / dot(dir, dir)); \n"
-            "        if (scale < 0.) scale = 1.; \n"
+            "        highp float numerator = dot(dir, ndir * pos.w * pos.w); \n"
+            "        highp float scale = 0.0; \n"
+            "        if (numerator < 0.0) \n"
+            "            scale = 1.0; \n"
+            "        else \n"
+            "            scale = min(1.0, numerator / dot(dir, dir)); \n"
             "        gl_Position += scale * delta; \n"
             "    } \n"